diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..5d716cc238f0e8cba315874cf7b6378e4fe7ca61 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,33 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zc0axaef/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/episode_rosbags/episode_1_2025-01-04-20-44-19.bag filter=lfs diff=lfs merge=lfs -text +y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/4eqzqa4m/.hydra/config.yaml b/4eqzqa4m/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/4eqzqa4m/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/4eqzqa4m/.hydra/hydra.yaml b/4eqzqa4m/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..715895ddefd80bb628880fd13051f3bd7a12f009 --- /dev/null +++ b/4eqzqa4m/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/4eqzqa4m + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/4eqzqa4m/.hydra/overrides.yaml b/4eqzqa4m/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/4eqzqa4m/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/4eqzqa4m/eval_robot.log b/4eqzqa4m/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b403a33dc74374448ccecdbc81d3440f888f23f2 --- /dev/null +++ b/4eqzqa4m/eval_robot.log @@ -0,0 +1,7 @@ +[2025-01-03 20:23:15,016][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:23:15,020][rospy.core][INFO] - signal_shutdown [atexit] diff --git a/bjqhpdkd/.hydra/config.yaml b/bjqhpdkd/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/bjqhpdkd/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/bjqhpdkd/.hydra/hydra.yaml b/bjqhpdkd/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8a1634a3ae7a8ae98442a03e39675f82fc84da7 --- /dev/null +++ b/bjqhpdkd/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/bjqhpdkd + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/bjqhpdkd/.hydra/overrides.yaml b/bjqhpdkd/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/bjqhpdkd/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/bjqhpdkd/eval_policy.log b/bjqhpdkd/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..11202f8fd71cecf7b43dbbe8fcfc27a677133086 --- /dev/null +++ b/bjqhpdkd/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 22:20:38,631][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:20:38,634][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 22:20:42,488][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fa809fa5f280b3c7a5decc30ef27a66fd5680eb --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 1 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1926_1' +wandb_notes: 1926_1_restarted_5 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/gdvjvd8r/.hydra/config.yaml b/gdvjvd8r/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/gdvjvd8r/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/gdvjvd8r/.hydra/hydra.yaml b/gdvjvd8r/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09b163cc9bde3fa0dd39239a47a7090970e6b06f --- /dev/null +++ b/gdvjvd8r/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/gdvjvd8r + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/gdvjvd8r/.hydra/overrides.yaml b/gdvjvd8r/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/gdvjvd8r/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/gdvjvd8r/eval_policy.log b/gdvjvd8r/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..6f986ef9af9eaf1f3a5c469e5ed791bb1f43cb3d --- /dev/null +++ b/gdvjvd8r/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 21:28:45,472][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:28:45,475][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 21:28:49,550][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/jrll14xh/.hydra/config.yaml b/jrll14xh/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/jrll14xh/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/jrll14xh/.hydra/hydra.yaml b/jrll14xh/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cea9fc1a6d7ea35f0f99d307ded4598773396743 --- /dev/null +++ b/jrll14xh/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/jrll14xh + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/jrll14xh/.hydra/overrides.yaml b/jrll14xh/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/jrll14xh/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/jrll14xh/eval_policy.log b/jrll14xh/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..9280ae0609b41ad770f53757cb620241e8f6d091 --- /dev/null +++ b/jrll14xh/eval_policy.log @@ -0,0 +1,7 @@ +[2025-01-03 20:23:12,469][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:23:12,474][rospy.core][INFO] - signal_shutdown [atexit] diff --git a/jrmd8cpt/.hydra/config.yaml b/jrmd8cpt/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/jrmd8cpt/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/jrmd8cpt/.hydra/hydra.yaml b/jrmd8cpt/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96ed6091deff86b26e480f84049ca8d3ae63feaf --- /dev/null +++ b/jrmd8cpt/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/jrmd8cpt + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/jrmd8cpt/.hydra/overrides.yaml b/jrmd8cpt/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/jrmd8cpt/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/jrmd8cpt/eval_policy.log b/jrmd8cpt/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..9f3bba8895b2b174a6407334f2d36626b2997b5f --- /dev/null +++ b/jrmd8cpt/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 21:12:58,368][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 21:12:58,372][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 21:13:02,291][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/p96n5ha2/.hydra/config.yaml b/p96n5ha2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/p96n5ha2/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/p96n5ha2/.hydra/hydra.yaml b/p96n5ha2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb6d42080addc1c8a063d1fa942c876ede13b171 --- /dev/null +++ b/p96n5ha2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/p96n5ha2/.hydra/overrides.yaml b/p96n5ha2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/p96n5ha2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/p96n5ha2/episode_rosbags/aligned_depth_to_color_K.npy b/p96n5ha2/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/p96n5ha2/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/p96n5ha2/episode_rosbags/cam_tf_world.npy b/p96n5ha2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/p96n5ha2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/p96n5ha2/episode_rosbags/color_K.npy b/p96n5ha2/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/p96n5ha2/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/p96n5ha2/episode_rosbags/depth_K.npy b/p96n5ha2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/p96n5ha2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/p96n5ha2/eval_robot.log b/p96n5ha2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0ad0f17b957350d24003e132a573c688892440ec --- /dev/null +++ b/p96n5ha2/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:58:25,259][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:58:25,262][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/p96n5ha2/eval_video/0_eval.mp4 b/p96n5ha2/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39df809a01fbc5a3104e94bf6b493dd0d2d99fbb --- /dev/null +++ b/p96n5ha2/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e4793e12534ec85ec6520ee2458c3039cb75c9463629819749011340205e24a +size 1584649 diff --git a/p96n5ha2/eval_video/1_eval.mp4 b/p96n5ha2/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9f89f2ccc79abf1207c1efd90737bd2c469b16e --- /dev/null +++ b/p96n5ha2/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ea4408164b8a140674496ba6629b16413df452969c28f8b7fb74e7b7c1f263 +size 1507254 diff --git a/p96n5ha2/eval_video/2_eval.mp4 b/p96n5ha2/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8192dc2c734e4227594023b4c52c750504a62a5b --- /dev/null +++ b/p96n5ha2/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:352b7f510e51bce389935d029bf3e54435b606213e8f5e38be4dba2eeb3cf22f +size 1556281 diff --git a/p96n5ha2/eval_video/3_eval.mp4 b/p96n5ha2/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fe5121a85c1f6335ee36f506dd0a970daf08937 --- /dev/null +++ b/p96n5ha2/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0172777c44cdec6ec773e208a913b0c9360554e00d010ab67a9ec13f40c91643 +size 1571685 diff --git a/p96n5ha2/eval_video/4_eval.mp4 b/p96n5ha2/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbfa18f5b5597b45d93a3e9bacb52862f9a29acb --- /dev/null +++ b/p96n5ha2/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6398d8b01df3925cc7cf3af337da5a7b49b62f423550339e0e3f74e019f1b7cf +size 1568166 diff --git a/p96n5ha2/tb/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 b/p96n5ha2/tb/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 new file mode 100644 index 0000000000000000000000000000000000000000..4d721c2a9236ad7823f1f02b65f5a6d58cccafbe --- /dev/null +++ b/p96n5ha2/tb/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62d59f3ecc23b44d281560039f9cfca9e8350e8ecf3d4b04285936840195a442 +size 6539 diff --git a/p96n5ha2/wandb/debug-internal.log b/p96n5ha2/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..704fd74b1dbf5b64939a48971fbfe6218e568eb0 --- /dev/null +++ b/p96n5ha2/wandb/debug-internal.log @@ -0,0 +1,871 @@ +2025-01-03 22:58:30,993 INFO StreamThr :1990615 [internal.py:wandb_internal():85] W&B internal server running at pid: 1990615, started at: 2025-01-03 22:58:30.993081 +2025-01-03 22:58:30,995 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:58:31,006 INFO WriterThread:1990615 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/run-p96n5ha2.wandb +2025-01-03 22:58:31,007 DEBUG SenderThread:1990615 [sender.py:send():379] send: header +2025-01-03 22:58:31,014 DEBUG SenderThread:1990615 [sender.py:send():379] send: run +2025-01-03 22:58:31,248 INFO SenderThread:1990615 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files +2025-01-03 22:58:31,248 INFO SenderThread:1990615 [sender.py:_start_run_threads():1188] run started: p96n5ha2 with start time 1735963110.993816 +2025-01-03 22:58:31,264 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:58:31,264 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:58:31,308 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:58:31,313 DEBUG HandlerThread:1990615 [system_info.py:__init__():26] System info init +2025-01-03 22:58:31,313 DEBUG HandlerThread:1990615 [system_info.py:__init__():41] System info init done +2025-01-03 22:58:31,313 INFO HandlerThread:1990615 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:58:31,313 INFO SystemMonitor:1990615 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:58:31,313 INFO HandlerThread:1990615 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:58:31,314 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:58:31,315 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:58:31,315 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:58:31,316 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:58:31,316 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:58:31,326 DEBUG HandlerThread:1990615 [system_info.py:probe():152] Probing system +2025-01-03 22:58:31,327 DEBUG HandlerThread:1990615 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_info.py:probe():200] Probing system done +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:58:31.326489', 'startedAt': '2025-01-04T03:58:30.982474', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.60125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 545.6132011413574}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:58:31,332 INFO HandlerThread:1990615 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:58:31,332 INFO HandlerThread:1990615 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_info.py:_save_code():44] Saving code +2025-01-03 22:58:31,334 DEBUG HandlerThread:1990615 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:58:31,334 DEBUG HandlerThread:1990615 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:58:31,400 DEBUG HandlerThread:1990615 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:58:31,400 INFO HandlerThread:1990615 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:58:31,404 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 22:58:31,404 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:58:31,404 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:58:31,404 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:58:31,635 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:58:31,635 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:58:31,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:58:31,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:58:31,665 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/9bn89vpn-wandb-metadata.json +2025-01-03 22:58:31,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:31,677 INFO HandlerThread:1990615 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/tb" + save: true +} +control { + relay_id: "13bbaf2b688b4ddfae70d62fcaa19845" +} +_info { + stream_id: "p96n5ha2" +} + +2025-01-03 22:58:31,677 DEBUG HandlerThread:1990615 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:58:31,681 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/bylcaskx-diff.patch +2025-01-03 22:58:31,705 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ds3q92zq-code/FISH/eval_robot.py +2025-01-03 22:58:31,745 DEBUG SenderThread:1990615 [sender.py:send():379] send: telemetry +2025-01-03 22:58:31,745 DEBUG SenderThread:1990615 [sender.py:send():379] send: tbrecord +2025-01-03 22:58:31,746 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 22:58:31,746 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 with policy live +2025-01-03 22:58:31,945 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/70ku7los-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH/eval_robot.py +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/diff.patch +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-metadata.json +2025-01-03 22:58:32,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH +2025-01-03 22:58:32,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code +2025-01-03 22:58:32,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:33,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:34,249 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:34,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:35,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:36,604 DEBUG SenderThread:1990615 [sender.py:send():379] send: config +2025-01-03 22:58:36,605 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:36,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:37,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:38,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:38,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:39,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:40,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:40,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:41,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:41,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:42,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:42,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:43,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:44,252 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:44,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:45,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:46,253 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:46,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:58:46,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:58:46,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:46,776 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:47,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:48,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:49,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:50,254 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:50,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:51,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:52,254 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:52,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:52,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:53,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:54,255 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:54,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:55,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:56,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:57,641 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:58,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:58,641 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:59,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:00,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:01,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:01,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:01,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:02,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:03,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:03,763 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:04,258 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml +2025-01-03 22:59:04,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:05,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:06,258 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:06,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:07,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:08,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:09,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:09,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:10,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:11,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:12,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:13,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:14,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:15,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:15,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:16,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:16,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:16,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:17,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:18,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:19,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:20,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:20,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:21,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:22,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:23,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:24,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:25,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:25,976 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:26,646 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:26,944 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 22:59:26,944 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 with policy now +2025-01-03 22:59:26,944 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:59:26,945 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 22:59:26,945 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:59:26,946 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:59:27,220 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/lvrgu8ea-media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 22:59:27,263 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos +2025-01-03 22:59:27,585 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/7mcsnjoi-media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 22:59:27,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:28,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:28,646 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:29,646 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:30,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:31,317 DEBUG SystemMonitor:1990615 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:59:31,317 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 22:59:31,317 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:31,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:31,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:31,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:32,265 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:32,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:33,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:34,266 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:34,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:35,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:36,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:36,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:37,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:38,267 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:59:38,547 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/2zcywnyn-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:59:38,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:39,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:40,268 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:40,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:41,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:41,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:42,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:43,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:44,270 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:44,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:45,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:46,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:46,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:46,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:46,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:47,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:48,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:49,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:50,272 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:50,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:51,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:52,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:52,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:53,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:54,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:55,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:56,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:57,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:57,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:58,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:59,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:00,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:01,318 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:00:01,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:01,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:01,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:02,275 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:02,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:03,581 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:03,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:04,276 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:04,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:05,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:06,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:07,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:08,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:08,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:09,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:10,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:11,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:12,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:13,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:14,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:14,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:15,653 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:15,989 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:00:15,989 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:00:15,990 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 with policy now +2025-01-03 23:00:15,991 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:00:15,991 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:00:16,007 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:00:16,265 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/jq3s83nx-media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:00:16,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:16,649 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:16,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:16,664 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/4huznejr-media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:00:17,655 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:18,281 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:18,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:19,655 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:20,129 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:20,281 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:20,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:21,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:22,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:23,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:24,283 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:24,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:25,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:25,910 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:26,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:27,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:28,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:29,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:30,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:31,318 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:00:31,319 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:31,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:31,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:31,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:32,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:33,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:34,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:35,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:36,641 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:36,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:37,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:38,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:39,102 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:00:39,102 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:00:39,103 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:00:39,287 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:00:39,288 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:00:39,530 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/wispqitq-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:00:39,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:40,288 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:40,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:41,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:42,053 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:42,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:43,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:44,289 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:44,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:45,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:46,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:46,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:46,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:47,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:47,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:48,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:49,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:50,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:51,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:52,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:53,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:53,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:54,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:55,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:56,293 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:56,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:57,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:58,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:59,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:59,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:00,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:01,319 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:01:01,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:01,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:01,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:02,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:03,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:04,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:04,703 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:05,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:06,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:07,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:08,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:09,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:10,330 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:10,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:11,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:12,298 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:12,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:13,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:14,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:15,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:15,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:16,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:16,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:16,685 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:17,668 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:18,203 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:01:18,203 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 with policy now +2025-01-03 23:01:18,204 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:01:18,204 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:01:18,204 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:01:18,217 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:01:18,300 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:01:18,300 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:18,300 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:01:18,301 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:01:18,483 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/97lh360m-media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:01:18,664 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:18,797 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/iut49r7y-media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:01:19,665 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:20,301 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:20,667 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:21,240 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:21,668 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:22,672 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:23,671 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:24,302 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:24,669 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:25,670 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:26,303 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:26,573 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:26,671 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:27,303 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:01:27,672 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:27,939 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:01:27,939 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:01:27,939 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:01:28,304 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:01:28,304 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:28,673 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:29,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:30,304 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:30,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:31,319 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:01:31,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:31,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:31,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:31,782 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:32,305 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:32,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:33,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:34,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:35,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:36,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:37,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:37,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:38,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:39,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:40,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:41,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:42,309 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:42,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:43,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:43,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:44,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:45,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:46,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:46,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:46,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:47,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:48,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:48,743 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:49,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:50,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:51,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:52,312 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:52,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:53,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:54,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:54,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:55,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:56,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:57,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:58,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:59,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:59,869 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:00,315 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:00,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:01,320 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:02:01,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:01,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:01,685 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:02,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:03,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:04,317 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:04,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:05,269 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:05,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:06,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:06,857 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:02:06,857 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 with policy now +2025-01-03 23:02:06,857 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:02:06,858 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:02:06,858 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:02:06,874 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:02:07,165 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/0kgd9hj1-media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:02:07,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:02:07,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:02:07,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:02:07,581 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/xg6ewz28-media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:02:07,687 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:08,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:08,684 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:09,688 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:10,404 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:10,691 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:11,685 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:12,320 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:12,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:13,688 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:14,320 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:14,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:15,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:16,198 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:16,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:16,644 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:16,690 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:17,690 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:18,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:19,689 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:20,697 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:21,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:21,690 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:22,696 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:23,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:24,696 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:25,696 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:26,695 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:26,907 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:27,010 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:02:27,010 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:02:27,011 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:02:27,325 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:02:27,325 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:02:27,598 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ztmvhhzo-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:02:27,694 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:28,325 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:28,701 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:29,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:30,326 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:30,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:31,320 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:02:31,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:31,640 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:31,698 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:32,326 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:32,698 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:32,745 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:33,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:34,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:35,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:36,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:37,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:38,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:38,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:39,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:40,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:41,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:42,330 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:42,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:43,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:44,329 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:44,331 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:44,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:45,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:46,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:46,640 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:46,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:47,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:48,701 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:49,701 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:50,265 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:50,332 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:50,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:51,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:52,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:53,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:54,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:55,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:55,703 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:56,705 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:57,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:58,335 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:58,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:59,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:00,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:00,873 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:01,321 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:03:01,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:01,640 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:01,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:02,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:03,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:04,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:05,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:06,468 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:06,711 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:07,009 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:03:07,009 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 with policy now +2025-01-03 23:03:07,016 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:03:07,016 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:03:07,016 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:03:07,017 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:03:07,287 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/mepe3108-media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:03:07,338 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:03:07,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:03:07,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:03:07,714 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:08,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:08,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:09,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:09,431 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/o88zduvg-media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:03:09,715 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:10,724 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:11,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:11,715 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:12,340 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:12,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:13,715 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:14,341 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:14,719 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:15,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:16,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:16,649 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:16,719 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:16,784 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:17,732 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:18,733 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:19,729 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:20,736 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:21,738 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:22,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:22,742 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:23,748 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:24,736 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:25,740 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:26,746 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:27,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:27,743 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:28,745 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:29,742 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:30,745 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:31,321 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:03:31,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:31,651 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:31,742 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:32,746 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:32,749 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:33,744 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:34,750 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:35,756 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:36,746 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:37,749 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:38,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:38,750 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:39,751 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:40,748 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:41,753 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:42,761 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:43,758 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:44,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:44,766 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:45,769 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:46,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:46,652 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:46,763 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:47,767 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:48,767 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:49,765 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:49,771 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:50,772 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:51,776 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:52,780 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:53,783 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:54,776 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:55,669 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:55,784 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:56,781 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:57,787 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:58,791 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:59,788 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:00,794 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:01,322 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:04:01,323 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:01,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:04:01,648 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:04:01,794 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:02,800 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:03,792 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:04,793 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:05,795 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:06,128 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:06,128 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:06,129 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:06,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:06,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:06,634 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ptizgtqy-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:06,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:06,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:04:06,717 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:04:06,791 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:06,881 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:04:07,174 INFO wandb-upload_1:1990615 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp79rm9ki7 +2025-01-03 23:04:07,302 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:04:07,310 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media +2025-01-03 23:04:07,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table +2025-01-03 23:04:07,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval +2025-01-03 23:04:07,774 INFO SenderThread:1990615 [sender.py:send_request_log_artifact():1525] logged artifact run-p96n5ha2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwOTIzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MzE5NQ==', 'latestArtifact': None}} +2025-01-03 23:04:07,774 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:04:07,774 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json with policy now +2025-01-03 23:04:07,775 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:07,775 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,775 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,776 DEBUG SenderThread:1990615 [sender.py:send():379] send: telemetry +2025-01-03 23:04:07,776 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,779 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,780 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,780 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,780 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,781 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,781 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,781 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,781 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,782 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,782 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,782 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,782 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,783 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,783 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,783 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,783 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,784 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,784 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,784 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,784 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,784 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,785 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:04:07,792 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:07,956 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/0li4uzqa-media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:08,002 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/wdmcikau-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:08,170 INFO wandb-upload_1:1990615 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvz5epteu +2025-01-03 23:04:08,245 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/wqgczt7x-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:08,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:08,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:08,607 INFO SenderThread:1990615 [sender.py:send_request_log_artifact():1525] logged artifact run-p96n5ha2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwOTI0OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MzE5Ng==', 'latestArtifact': None}} +2025-01-03 23:04:08,608 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:04:08,608 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json with policy now +2025-01-03 23:04:08,608 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:08,609 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:08,609 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:08,610 DEBUG SenderThread:1990615 [sender.py:send():379] send: exit +2025-01-03 23:04:08,610 INFO SenderThread:1990615 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 23:04:08,610 INFO SenderThread:1990615 [sender.py:send_exit():588] handling runtime: 337 +2025-01-03 23:04:08,613 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:08,613 INFO SenderThread:1990615 [sender.py:send_exit():594] send defer +2025-01-03 23:04:08,614 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,614 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 23:04:08,614 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 1 +2025-01-03 23:04:08,614 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,614 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 23:04:08,614 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 2 +2025-01-03 23:04:08,614 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,614 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 23:04:08,615 INFO HandlerThread:1990615 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 23:04:08,615 DEBUG SystemMonitor:1990615 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 23:04:08,615 DEBUG SystemMonitor:1990615 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 23:04:08,615 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 23:04:08,615 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 23:04:08,670 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 23:04:08,671 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 23:04:08,671 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined network monitor +2025-01-03 23:04:08,671 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,671 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 23:04:08,671 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 3 +2025-01-03 23:04:08,671 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:04:08,671 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,672 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 23:04:08,672 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,672 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 23:04:08,672 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 4 +2025-01-03 23:04:08,672 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,672 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 23:04:08,857 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/mled8cx1-media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:09,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:10,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:15,133 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:15,133 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:15,133 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 23:04:15,134 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,134 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 23:04:15,134 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 5 +2025-01-03 23:04:15,134 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:15,135 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:15,135 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:15,135 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:15,135 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:15,136 DEBUG SenderThread:1990615 [sender.py:send():379] send: summary +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:15,136 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 6 +2025-01-03 23:04:15,137 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:15,137 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 23:04:15,137 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,137 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 23:04:15,140 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:15,255 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 7 +2025-01-03 23:04:15,255 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:15,255 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 23:04:15,255 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,255 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 23:04:15,362 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:15,363 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml +2025-01-03 23:04:15,610 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:17,075 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 8 +2025-01-03 23:04:17,075 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:17,076 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:17,076 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 23:04:17,076 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:17,076 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 23:04:17,076 INFO SenderThread:1990615 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 23:04:17,078 INFO SenderThread:1990615 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 23:04:17,080 INFO SenderThread:1990615 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 23:04:17,095 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 9 +2025-01-03 23:04:17,095 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:17,095 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 23:04:17,096 DEBUG SenderThread:1990615 [sender.py:send():379] send: artifact +2025-01-03 23:04:17,363 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:17,427 INFO SenderThread:1990615 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'versionIndex': 414}}} +2025-01-03 23:04:17,427 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:17,428 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 23:04:17,428 INFO SenderThread:1990615 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 23:04:17,611 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml config.yaml +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json wandb-summary.json +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt requirements.txt +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-metadata.json wandb-metadata.json +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log output.log +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/diff.patch diff.patch +2025-01-03 23:04:18,367 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 10 +2025-01-03 23:04:18,375 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:18,375 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:18,375 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 23:04:18,375 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:18,375 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 23:04:18,375 INFO SenderThread:1990615 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:04:18,556 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:18,581 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt +2025-01-03 23:04:18,600 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml +2025-01-03 23:04:18,612 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:18,612 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:18,620 INFO wandb-upload_4:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:18,662 INFO wandb-upload_3:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ac16q5jn-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:18,863 INFO Thread-11 (_thread_body):1990615 [sender.py:transition_state():614] send defer: 11 +2025-01-03 23:04:18,863 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:18,863 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 23:04:18,863 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:18,863 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 23:04:18,863 INFO SenderThread:1990615 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 23:04:18,865 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 12 +2025-01-03 23:04:18,865 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:18,865 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 23:04:18,865 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:18,865 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 23:04:18,865 INFO SenderThread:1990615 [file_stream.py:finish():601] file stream finish called +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [file_stream.py:finish():605] file stream finish is done +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 13 +2025-01-03 23:04:19,060 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:19,060 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 23:04:19,060 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 14 +2025-01-03 23:04:19,060 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:19,061 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 23:04:19,061 DEBUG SenderThread:1990615 [sender.py:send():379] send: final +2025-01-03 23:04:19,061 DEBUG SenderThread:1990615 [sender.py:send():379] send: footer +2025-01-03 23:04:19,061 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:19,061 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 23:04:19,062 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:19,062 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:19,062 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 23:04:19,062 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: server_info +2025-01-03 23:04:19,064 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:19,064 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 23:04:19,064 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 23:04:19,065 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:19,126 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:19,127 INFO MainThread:1990615 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 23:04:19,127 INFO MainThread:1990615 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 23:04:19,128 INFO MainThread:1990615 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 23:04:19,128 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 23:04:19,128 INFO HandlerThread:1990615 [handler.py:finish():882] shutting down handler +2025-01-03 23:04:20,065 INFO WriterThread:1990615 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/run-p96n5ha2.wandb +2025-01-03 23:04:20,127 INFO SenderThread:1990615 [sender.py:finish():1615] shutting down sender +2025-01-03 23:04:20,127 INFO SenderThread:1990615 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:04:20,127 INFO SenderThread:1990615 [file_pusher.py:join():175] waiting for file pusher diff --git a/p96n5ha2/wandb/debug.log b/p96n5ha2/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..610caa6894eea59401dec5141f1286b8c296774f --- /dev/null +++ b/p96n5ha2/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Configure stats pid to 1990442 +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/settings +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug.log +2025-01-03 22:58:30,987 INFO MainThread:1990442 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug-internal.log +2025-01-03 22:58:30,987 INFO MainThread:1990442 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:58:30,987 INFO MainThread:1990442 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2'} +2025-01-03 22:58:30,988 INFO MainThread:1990442 [wandb_init.py:init():619] starting backend +2025-01-03 22:58:30,988 INFO MainThread:1990442 [wandb_init.py:init():623] setting up manager +2025-01-03 22:58:30,992 INFO MainThread:1990442 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:58:30,993 INFO MainThread:1990442 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:58:31,005 INFO MainThread:1990442 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:58:31,012 INFO MainThread:1990442 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:58:31,263 INFO MainThread:1990442 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:58:31,303 INFO MainThread:1990442 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:58:31,303 INFO MainThread:1990442 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:58:31,637 INFO MainThread:1990442 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:58:31,638 INFO MainThread:1990442 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/tb, True +2025-01-03 22:58:36,604 INFO MainThread:1990442 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 23:04:20,129 WARNING MsgRouterThr:1990442 [router.py:message_loop():77] message_loop has been closed diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH/eval_robot.py b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f97d487070930b7ce8ae04c466ae707564f8b98 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735963110 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/diff.patch b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..91d1314b0c81ca7ced2bf379f5d1a3bb24013b63 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.20694816118379347, 0.9527384935904063, 0.20744311815390934, 0.9364891498407951, 0.2632499635693074, 0.8492476194398473, 0.24815547471314967, 0.8085234164574604]]} \ No newline at end of file diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json new file mode 100644 index 0000000000000000000000000000000000000000..834fa098d61080ac8a2af619db0fcec154bba93a --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[16, 30, 0.5333333333333333, 0.3427325189428152, 0.7259325384641823, 0.3498348743023795, 0.6953116207924273, 0.3724039420740689, 0.6802478581306451, 0.4301501289149747, 0.6539621799876325]]} \ No newline at end of file diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39df809a01fbc5a3104e94bf6b493dd0d2d99fbb --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e4793e12534ec85ec6520ee2458c3039cb75c9463629819749011340205e24a +size 1584649 diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9f89f2ccc79abf1207c1efd90737bd2c469b16e --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ea4408164b8a140674496ba6629b16413df452969c28f8b7fb74e7b7c1f263 +size 1507254 diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8192dc2c734e4227594023b4c52c750504a62a5b --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:352b7f510e51bce389935d029bf3e54435b606213e8f5e38be4dba2eeb3cf22f +size 1556281 diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fe5121a85c1f6335ee36f506dd0a970daf08937 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0172777c44cdec6ec773e208a913b0c9360554e00d010ab67a9ec13f40c91643 +size 1571685 diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbfa18f5b5597b45d93a3e9bacb52862f9a29acb --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6398d8b01df3925cc7cf3af337da5a7b49b62f423550339e0e3f74e019f1b7cf +size 1568166 diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..49b503ded797536ecd64609eafa7077c5ad0ec96 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735963116.605853]: resetting environment +[INFO] [1735963116.613068]: cleared current plan +[INFO] [1735963116.614200]: moving to home +[INFO] [1735963117.616613]: reached home +[INFO] [1735963117.618023]: reset action history +[INFO] [1735963120.327955]: environment reset +Starting episode 0 +[INFO] [1735963120.332555]: resetting environment +[INFO] [1735963120.338644]: cleared current plan +[INFO] [1735963120.340014]: moving to home +[INFO] [1735963121.342666]: reached home +[INFO] [1735963121.343900]: reset action history +[INFO] [1735963124.052095]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735963127.340186]: resetting environment +[INFO] [1735963127.346722]: cleared current plan +[INFO] [1735963127.347811]: moving to home +[INFO] [1735963128.349337]: reached home +[INFO] [1735963128.350634]: reset action history +[INFO] [1735963131.059216]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/episode_rosbags/episode_0_2025-01-03-22-58-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1990442.log +started roslaunch server http://158.130.50.37:34707/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1990744] +process[print_text-2]: started with pid [1990745] +process[rosbag_record-3]: started with pid [1990746] +started bagging! +For topic gripper_width: timestamp difference is -105547023 for nearest: 1735963144461516070 - target: 1735963144567063093 at idx 188 +[INFO] [1735963165.975778]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735963166.710581]: Stored episode 1. +[INFO] [1735963166.710842]: Saving video... +[INFO] [1735963166.925253]: Video saved! +Starting episode 1 +[INFO] [1735963166.944940]: resetting environment +[INFO] [1735963166.952869]: cleared current plan +[INFO] [1735963166.953163]: moving to home +[INFO] [1735963170.456658]: reached home +[INFO] [1735963170.457115]: reset action history +[INFO] [1735963173.168247]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963178.063614]: resetting environment +[INFO] [1735963178.068279]: cleared current plan +[INFO] [1735963178.068444]: moving to home +[INFO] [1735963179.069249]: reached home +[INFO] [1735963179.069650]: reset action history +[INFO] [1735963181.777369]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/episode_rosbags/episode_1_2025-01-03-22-59-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1990442.log +started roslaunch server http://158.130.50.37:36671/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1990870] +process[print_text-5]: started with pid [1990894] +process[rosbag_record-6]: started with pid [1990895] +started bagging! +For topic gripper_width: timestamp difference is -103969261 for nearest: 1735963187761516884 - target: 1735963187865486145 at idx 82 +For topic gripper_width: timestamp difference is 105273072 for nearest: 1735963199428182427 - target: 1735963199322909355 at idx 226 +For topic gripper_width: timestamp difference is 107829136 for nearest: 1735963202528207583 - target: 1735963202420378447 at idx 249 +preempting episode +[INFO] [1735963214.895187]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735963215.637389]: Stored episode 2. +[INFO] [1735963215.637668]: Saving video... +[INFO] [1735963215.942272]: Video saved! +Starting episode 2 +[INFO] [1735963215.967470]: resetting environment +[INFO] [1735963215.980747]: cleared current plan +[INFO] [1735963215.981027]: moving to home +[INFO] [1735963219.084946]: reached home +[INFO] [1735963219.090258]: reset action history +[INFO] [1735963221.864906]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963238.328332]: resetting environment +[INFO] [1735963238.340118]: cleared current plan +[INFO] [1735963238.341284]: moving to home +[INFO] [1735963239.343256]: reached home +[INFO] [1735963239.343716]: reset action history +[INFO] [1735963242.052695]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/episode_rosbags/episode_2_2025-01-03-23-00-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1990442.log +started roslaunch server http://158.130.50.37:36973/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1991047] +process[print_text-8]: started with pid [1991048] +process[rosbag_record-9]: started with pid [1991049] +started bagging! +For topic gripper_width: timestamp difference is 116341523 for nearest: 1735963254428184680 - target: 1735963254311843157 at idx 171 +For topic gripper_width: timestamp difference is 129234826 for nearest: 1735963270328181779 - target: 1735963270198946953 at idx 249 +For topic gripper_width: timestamp difference is 101083382 for nearest: 1735963276828173502 - target: 1735963276727090120 at idx 249 +[INFO] [1735963277.072950]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735963277.894449]: Stored episode 3. +[INFO] [1735963277.896432]: Saving video... +[INFO] [1735963278.157827]: Video saved! +Starting episode 3 +[INFO] [1735963278.208383]: resetting environment +[INFO] [1735963278.228942]: cleared current plan +[INFO] [1735963278.234535]: moving to home +[INFO] [1735963281.745425]: reached home +[INFO] [1735963281.747333]: reset action history +[INFO] [1735963284.527932]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963287.011626]: resetting environment +[INFO] [1735963287.028859]: cleared current plan +[INFO] [1735963287.033757]: moving to home +[INFO] [1735963288.042753]: reached home +[INFO] [1735963288.047735]: reset action history +[INFO] [1735963290.818521]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/episode_rosbags/episode_3_2025-01-03-23-01-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1990442.log +started roslaunch server http://158.130.50.37:32909/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1991173] +process[print_text-11]: started with pid [1991174] +process[rosbag_record-12]: started with pid [1991198] +started bagging! +For topic gripper_width: timestamp difference is -107787871 for nearest: 1735963299661515212 - target: 1735963299769303083 at idx 130 +For topic gripper_width: timestamp difference is -104833640 for nearest: 1735963310561514579 - target: 1735963310666348219 at idx 249 +For topic gripper_width: timestamp difference is 100628261 for nearest: 1735963317528183822 - target: 1735963317427555561 at idx 249 +For topic gripper_width: timestamp difference is 101715531 for nearest: 1735963318828184094 - target: 1735963318726468563 at idx 249 +For topic gripper_width: timestamp difference is 137755032 for nearest: 1735963322228181000 - target: 1735963322090425968 at idx 249 +[INFO] [1735963325.652343]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735963326.405719]: Stored episode 4. +[INFO] [1735963326.405988]: Saving video... +[INFO] [1735963326.811771]: Video saved! +Starting episode 4 +[INFO] [1735963326.864602]: resetting environment +[INFO] [1735963326.892542]: cleared current plan +[INFO] [1735963326.893237]: moving to home +[INFO] [1735963330.403923]: reached home +[INFO] [1735963330.406896]: reset action history +[INFO] [1735963333.151617]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963346.934791]: resetting environment +[INFO] [1735963346.968691]: cleared current plan +[INFO] [1735963346.971782]: moving to home +[INFO] [1735963347.975488]: reached home +[INFO] [1735963347.980129]: reset action history +[INFO] [1735963350.763784]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/episode_rosbags/episode_4_2025-01-03-23-02-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1990442.log +started roslaunch server http://158.130.50.37:39353/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1991317] +process[print_text-14]: started with pid [1991318] +process[rosbag_record-15]: started with pid [1991342] +started bagging! +For topic gripper_width: timestamp difference is 107776627 for nearest: 1735963360128183350 - target: 1735963360020406723 at idx 138 +For topic gripper_width: timestamp difference is 103417518 for nearest: 1735963362228275182 - target: 1735963362124857664 at idx 167 +For topic gripper_width: timestamp difference is 104150521 for nearest: 1735963363261513459 - target: 1735963363157362938 at idx 180 +For topic gripper_width: timestamp difference is -158482839 for nearest: 1735963367961514662 - target: 1735963368119997501 at idx 245 +For topic gripper_width: timestamp difference is 115162601 for nearest: 1735963376828182210 - target: 1735963376713019609 at idx 249 +[INFO] [1735963385.628906]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735963386.466889]: Stored episode 5. +[INFO] [1735963386.469401]: Saving video... +[INFO] [1735963386.961037]: Video saved! +[INFO] [1735963387.034631]: resetting environment +[INFO] [1735963387.062143]: cleared current plan +[INFO] [1735963387.067738]: moving to home +[WARN] [1735963386.973687]: Plan exhausted +[WARN] [1735963387.013977]: Plan exhausted +[WARN] [1735963387.055570]: Plan exhausted +[INFO] [1735963390.676856]: reached home +[INFO] [1735963390.679460]: reset action history +[INFO] [1735963393.526067]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-metadata.json b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..aaf953699804317d8c27e72bb1682af3ab1b08e3 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:58:31.326489", + "startedAt": "2025-01-04T03:58:30.982474", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.60125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 545.6132011413574 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..902524c2358e9872d65d0af29886054bc556fb82 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "5e4793e12534ec85ec6520ee2458c3039cb75c9463629819749011340205e24a", "size": 1584649, "path": "media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4"}, "global_step": 658, "_timestamp": 1735963447.200469, "_runtime": 336.20665311813354, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "d3ea4408164b8a140674496ba6629b16413df452969c28f8b7fb74e7b7c1f263", "size": 1507254, "path": "media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4"}, "eval/num_success": 3.0, "episode": 30.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.2069481611251831, "eval/success_rate_ub_0.01": 0.9527384638786316, "eval/success_rate_lb_0.025": 0.20744311809539795, "eval/success_rate_ub_0.025": 0.9364891648292542, "eval/success_rate_lb_0.05": 0.2632499635219574, "eval/success_rate_ub_0.05": 0.8492476344108582, "eval/success_rate_lb_0.1": 0.24815547466278076, "eval/success_rate_ub_0.1": 0.808523416519165, "eval/2_eval": {"_type": "video-file", "sha256": "352b7f510e51bce389935d029bf3e54435b606213e8f5e38be4dba2eeb3cf22f", "size": 1556281, "path": "media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "0172777c44cdec6ec773e208a913b0c9360554e00d010ab67a9ec13f40c91643", "size": 1571685, "path": "media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "6398d8b01df3925cc7cf3af337da5a7b49b62f423550339e0e3f74e019f1b7cf", "size": 1568166, "path": "media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "21a9f3410d4e4ea9386f8f8b05d066553e0d5981bed4f42ec4bc66fd0dcd2c14", "size": 401, "artifact_path": "wandb-client-artifact://ll3ypte2wdmowuht16k5jvotvugjouo63pbqj04vt8fz3mzs4lsgnr06s5wc64u39sa4hare64ql3i3ewx9g6gn75ljd6dno4jm8cx9xy7ovk0bf7uqy6wym7j73bjhl/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://rl9kkuy5zot4r8s5y5b8czv07mlfg8voi862r6dd0c5niyk2xzuw553u52r64867jwq85ah8n024j1rdir90gqhhcqwvr5x6ammdoikkfcp9556nh8g2bsiuidlk2vlz:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 16, "total_num_episodes": 30, "total_success_rate": 0.5333333333333333, "total_success_rate_lb_0.01": 0.3427325189428152, "total_success_rate_ub_0.01": 0.7259325384641823, "total_success_rate_lb_0.025": 0.3498348743023795, "total_success_rate_ub_0.025": 0.6953116207924273, "total_success_rate_lb_0.05": 0.3724039420740689, "total_success_rate_ub_0.05": 0.6802478581306451, "total_success_rate_lb_0.1": 0.4301501289149747, "total_success_rate_ub_0.1": 0.6539621799876325, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "b44f9defe3679a919f054317ed5a8f71aaf22b7af502b82edfe93ff1537046ff", "size": 520, "artifact_path": "wandb-client-artifact://y3dxjszn5fvdh2ij0jay19arevy2h488ljbx99ljyq01sxscn29tjef7cykimducv61h0tp89fiaey02605ae05oscjut7242svrjpckihyv325atcha738ni0xic08b/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xw8bs57e4exxa5pc3vuvk3xmn7duqyapvc8wr30wvi2xc5hx9h8diisfso6eszvr8n6sqx24hz0iyxm4iupogrud188rox01xljkol6fvvptb5bs9mhwueoe9at3usbj:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 337}, "eval/total_success_rate": 0.5333333611488342, "eval/total_success_rate_lb_0.01": 0.3427325189113617, "eval/total_success_rate_ub_0.01": 0.7259325385093689, "eval/total_success_rate_lb_0.025": 0.3498348891735077, "eval/total_success_rate_ub_0.025": 0.6953116059303284, "eval/total_success_rate_lb_0.05": 0.37240394949913025, "eval/total_success_rate_ub_0.05": 0.6802478432655334, "eval/total_success_rate_lb_0.1": 0.4301501214504242, "eval/total_success_rate_ub_0.1": 0.6539621949195862} \ No newline at end of file diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug-internal.log b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..704fd74b1dbf5b64939a48971fbfe6218e568eb0 --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug-internal.log @@ -0,0 +1,871 @@ +2025-01-03 22:58:30,993 INFO StreamThr :1990615 [internal.py:wandb_internal():85] W&B internal server running at pid: 1990615, started at: 2025-01-03 22:58:30.993081 +2025-01-03 22:58:30,995 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:58:31,006 INFO WriterThread:1990615 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/run-p96n5ha2.wandb +2025-01-03 22:58:31,007 DEBUG SenderThread:1990615 [sender.py:send():379] send: header +2025-01-03 22:58:31,014 DEBUG SenderThread:1990615 [sender.py:send():379] send: run +2025-01-03 22:58:31,248 INFO SenderThread:1990615 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files +2025-01-03 22:58:31,248 INFO SenderThread:1990615 [sender.py:_start_run_threads():1188] run started: p96n5ha2 with start time 1735963110.993816 +2025-01-03 22:58:31,264 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:58:31,264 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:58:31,308 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:58:31,313 DEBUG HandlerThread:1990615 [system_info.py:__init__():26] System info init +2025-01-03 22:58:31,313 DEBUG HandlerThread:1990615 [system_info.py:__init__():41] System info init done +2025-01-03 22:58:31,313 INFO HandlerThread:1990615 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:58:31,313 INFO SystemMonitor:1990615 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:58:31,313 INFO HandlerThread:1990615 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:58:31,314 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:58:31,315 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:58:31,315 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:58:31,316 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:58:31,316 INFO SystemMonitor:1990615 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:58:31,326 DEBUG HandlerThread:1990615 [system_info.py:probe():152] Probing system +2025-01-03 22:58:31,327 DEBUG HandlerThread:1990615 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_info.py:probe():200] Probing system done +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:58:31.326489', 'startedAt': '2025-01-04T03:58:30.982474', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.60125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 545.6132011413574}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:58:31,332 INFO HandlerThread:1990615 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:58:31,332 INFO HandlerThread:1990615 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:58:31,332 DEBUG HandlerThread:1990615 [system_info.py:_save_code():44] Saving code +2025-01-03 22:58:31,334 DEBUG HandlerThread:1990615 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:58:31,334 DEBUG HandlerThread:1990615 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:58:31,400 DEBUG HandlerThread:1990615 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:58:31,400 INFO HandlerThread:1990615 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:58:31,404 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 22:58:31,404 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:58:31,404 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:58:31,404 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:58:31,635 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:58:31,635 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:58:31,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:58:31,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:58:31,665 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/9bn89vpn-wandb-metadata.json +2025-01-03 22:58:31,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:31,677 INFO HandlerThread:1990615 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/tb" + save: true +} +control { + relay_id: "13bbaf2b688b4ddfae70d62fcaa19845" +} +_info { + stream_id: "p96n5ha2" +} + +2025-01-03 22:58:31,677 DEBUG HandlerThread:1990615 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:58:31,681 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/bylcaskx-diff.patch +2025-01-03 22:58:31,705 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ds3q92zq-code/FISH/eval_robot.py +2025-01-03 22:58:31,745 DEBUG SenderThread:1990615 [sender.py:send():379] send: telemetry +2025-01-03 22:58:31,745 DEBUG SenderThread:1990615 [sender.py:send():379] send: tbrecord +2025-01-03 22:58:31,746 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 22:58:31,746 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 with policy live +2025-01-03 22:58:31,945 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/70ku7los-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH/eval_robot.py +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/diff.patch +2025-01-03 22:58:32,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-metadata.json +2025-01-03 22:58:32,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH +2025-01-03 22:58:32,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code +2025-01-03 22:58:32,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:33,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:34,249 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:34,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:35,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:36,604 DEBUG SenderThread:1990615 [sender.py:send():379] send: config +2025-01-03 22:58:36,605 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:36,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:37,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:38,250 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:38,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:39,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:40,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:40,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:41,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:41,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:42,251 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:42,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:43,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:44,252 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:44,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:45,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:46,253 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:46,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:58:46,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:58:46,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:46,776 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:47,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:48,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:49,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:50,254 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:50,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:51,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:52,254 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:52,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:52,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:53,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:54,255 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:58:54,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:55,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:56,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:57,641 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:58,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:58,641 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:59,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:00,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:01,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:01,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:01,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:02,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:03,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:03,763 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:04,258 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml +2025-01-03 22:59:04,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:05,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:06,258 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:06,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:07,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:08,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:09,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:09,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:10,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:11,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:12,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:13,643 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:14,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:15,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:15,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:16,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:16,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:16,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:17,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:18,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:19,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:20,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:20,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:21,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:22,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:23,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:24,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:25,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:25,976 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:26,646 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:26,944 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 22:59:26,944 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 with policy now +2025-01-03 22:59:26,944 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:59:26,945 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 22:59:26,945 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:59:26,946 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:59:27,220 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/lvrgu8ea-media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 22:59:27,263 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media +2025-01-03 22:59:27,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos +2025-01-03 22:59:27,585 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/7mcsnjoi-media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 22:59:27,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:28,264 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:28,646 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:29,646 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:30,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:31,317 DEBUG SystemMonitor:1990615 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:59:31,317 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 22:59:31,317 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:31,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:31,636 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:31,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:32,265 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:32,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:33,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:34,266 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:34,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:35,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:36,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:36,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:37,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:38,267 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:59:38,547 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/2zcywnyn-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 22:59:38,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:39,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:40,268 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:40,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:41,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:41,647 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:42,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:43,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:44,270 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:44,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:45,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:46,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:59:46,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:59:46,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:46,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:47,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:48,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:49,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:50,272 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 22:59:50,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:51,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:52,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:52,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:53,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:54,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:55,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:56,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:57,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:59:57,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:58,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:59:59,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:00,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:01,318 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:00:01,636 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:01,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:01,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:02,275 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:02,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:03,581 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:03,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:04,276 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:04,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:05,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:06,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:07,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:08,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:08,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:09,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:10,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:11,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:12,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:13,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:14,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:14,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:15,653 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:15,989 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:00:15,989 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:00:15,990 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 with policy now +2025-01-03 23:00:15,991 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:00:15,991 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:00:16,007 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:00:16,265 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/jq3s83nx-media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:00:16,280 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:00:16,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:16,649 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:16,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:16,664 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/4huznejr-media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:00:17,655 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:18,281 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:18,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:19,655 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:20,129 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:20,281 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:20,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:21,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:22,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:23,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:24,283 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:24,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:25,656 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:25,910 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:26,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:27,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:28,657 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:29,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:30,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:31,318 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:00:31,319 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:31,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:31,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:31,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:32,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:33,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:34,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:35,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:36,641 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:36,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:37,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:38,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:39,102 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:00:39,102 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:00:39,103 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:00:39,287 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:00:39,288 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:00:39,530 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/wispqitq-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:00:39,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:40,288 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:40,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:41,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:42,053 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:42,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:43,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:44,289 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:44,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:45,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:46,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:00:46,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:00:46,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:47,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:47,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:48,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:49,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:50,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:51,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:52,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:53,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:53,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:54,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:55,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:56,293 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:00:56,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:57,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:58,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:00:59,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:00:59,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:00,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:01,319 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:01:01,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:01,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:01,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:02,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:03,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:04,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:04,703 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:05,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:06,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:07,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:08,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:09,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:10,330 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:10,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:11,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:12,298 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:12,662 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:13,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:14,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:15,642 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:15,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:16,637 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:16,637 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:16,685 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:17,668 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:18,203 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:01:18,203 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 with policy now +2025-01-03 23:01:18,204 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:01:18,204 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:01:18,204 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:01:18,217 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:01:18,300 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:01:18,300 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:18,300 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:01:18,301 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:01:18,483 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/97lh360m-media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:01:18,664 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:18,797 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/iut49r7y-media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:01:19,665 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:20,301 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:20,667 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:21,240 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:21,668 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:22,672 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:23,671 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:24,302 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:24,669 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:25,670 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:26,303 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:26,573 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:26,671 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:27,303 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:01:27,672 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:27,939 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:01:27,939 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:01:27,939 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:01:28,304 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:01:28,304 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:28,673 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:29,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:30,304 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:30,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:31,319 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:01:31,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:31,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:31,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:31,782 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:32,305 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:32,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:33,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:34,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:35,674 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:36,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:37,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:37,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:38,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:39,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:40,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:41,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:42,309 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:42,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:43,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:43,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:44,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:45,675 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:46,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:01:46,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:01:46,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:47,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:48,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:48,743 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:49,676 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:50,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:51,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:52,312 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:01:52,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:53,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:54,645 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:01:54,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:55,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:56,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:57,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:58,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:59,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:01:59,869 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:00,315 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:00,677 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:01,320 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:02:01,638 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:01,639 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:01,685 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:02,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:03,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:04,317 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:04,678 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:05,269 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:05,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:06,681 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:06,857 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:02:06,857 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 with policy now +2025-01-03 23:02:06,857 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:02:06,858 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:02:06,858 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:02:06,874 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:02:07,165 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/0kgd9hj1-media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:02:07,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:02:07,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:02:07,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:02:07,581 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/xg6ewz28-media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:02:07,687 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:08,318 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:08,684 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:09,688 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:10,404 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:10,691 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:11,685 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:12,320 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:12,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:13,688 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:14,320 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:14,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:15,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:16,198 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:16,644 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:16,644 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:16,690 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:17,690 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:18,692 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:19,689 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:20,697 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:21,651 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:21,690 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:22,696 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:23,693 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:24,696 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:25,696 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:26,695 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:26,907 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:27,010 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:02:27,010 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:02:27,011 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:02:27,325 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:02:27,325 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:02:27,598 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ztmvhhzo-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:02:27,694 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:28,325 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:28,701 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:29,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:30,326 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:30,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:31,320 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:02:31,639 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:31,640 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:31,698 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:32,326 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:32,698 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:32,745 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:33,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:34,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:35,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:36,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:37,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:38,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:38,699 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:39,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:40,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:41,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:42,330 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:42,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:43,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:44,329 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:44,331 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:44,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:45,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:46,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:02:46,640 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:02:46,700 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:47,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:48,701 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:49,701 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:50,265 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:50,332 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:50,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:51,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:52,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:53,702 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:54,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:55,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:02:55,703 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:56,705 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:57,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:58,335 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:02:58,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:02:59,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:00,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:00,873 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:01,321 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:03:01,640 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:01,640 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:01,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:02,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:03,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:04,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:05,704 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:06,468 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:06,711 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:07,009 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:03:07,009 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 with policy now +2025-01-03 23:03:07,016 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:03:07,016 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:03:07,016 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:03:07,017 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:03:07,287 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/mepe3108-media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:03:07,338 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:03:07,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:03:07,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval +2025-01-03 23:03:07,714 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:08,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:08,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:09,339 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:09,431 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/o88zduvg-media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:03:09,715 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:10,724 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:11,659 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:11,715 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:12,340 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:12,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:13,715 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:14,341 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:03:14,719 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:15,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:16,649 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:16,649 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:16,719 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:16,784 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:17,732 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:18,733 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:19,729 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:20,736 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:21,738 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:22,658 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:22,742 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:23,748 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:24,736 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:25,740 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:26,746 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:27,661 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:27,743 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:28,745 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:29,742 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:30,745 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:31,321 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:03:31,650 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:31,651 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:31,742 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:32,746 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:32,749 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:33,744 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:34,750 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:35,756 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:36,746 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:37,749 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:38,663 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:38,750 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:39,751 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:40,748 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:41,753 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:42,761 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:43,758 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:44,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:44,766 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:45,769 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:46,652 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:03:46,652 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:03:46,763 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:47,767 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:48,767 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:49,765 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:49,771 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:50,772 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:51,776 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:52,780 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:53,783 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:54,776 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:55,669 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:03:55,784 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:56,781 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:57,787 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:58,791 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:03:59,788 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:00,794 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:01,322 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:04:01,323 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:01,648 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:04:01,648 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:04:01,794 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:02,800 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:03,792 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:04,793 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:05,795 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:06,128 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:06,128 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:06,129 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:06,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:06,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:06,634 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ptizgtqy-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:06,660 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:06,717 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:04:06,717 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:04:06,791 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:06,881 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:04:07,174 INFO wandb-upload_1:1990615 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp79rm9ki7 +2025-01-03 23:04:07,302 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:04:07,310 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:07,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media +2025-01-03 23:04:07,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table +2025-01-03 23:04:07,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval +2025-01-03 23:04:07,774 INFO SenderThread:1990615 [sender.py:send_request_log_artifact():1525] logged artifact run-p96n5ha2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwOTIzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MzE5NQ==', 'latestArtifact': None}} +2025-01-03 23:04:07,774 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:04:07,774 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json with policy now +2025-01-03 23:04:07,775 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:07,775 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,775 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,776 DEBUG SenderThread:1990615 [sender.py:send():379] send: telemetry +2025-01-03 23:04:07,776 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,779 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,780 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,780 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,780 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,781 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,781 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,781 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,781 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,782 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,782 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,782 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,782 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,783 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,783 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,783 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,783 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,784 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,784 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,784 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,784 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:07,784 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:07,785 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:04:07,792 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:07,956 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/0li4uzqa-media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:08,002 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/wdmcikau-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:08,170 INFO wandb-upload_1:1990615 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvz5epteu +2025-01-03 23:04:08,245 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/wqgczt7x-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:08,359 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:08,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:08,607 INFO SenderThread:1990615 [sender.py:send_request_log_artifact():1525] logged artifact run-p96n5ha2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwOTI0OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MzE5Ng==', 'latestArtifact': None}} +2025-01-03 23:04:08,608 DEBUG SenderThread:1990615 [sender.py:send():379] send: files +2025-01-03 23:04:08,608 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json with policy now +2025-01-03 23:04:08,608 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:08,609 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:08,609 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:08,610 DEBUG SenderThread:1990615 [sender.py:send():379] send: exit +2025-01-03 23:04:08,610 INFO SenderThread:1990615 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 23:04:08,610 INFO SenderThread:1990615 [sender.py:send_exit():588] handling runtime: 337 +2025-01-03 23:04:08,613 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:08,613 INFO SenderThread:1990615 [sender.py:send_exit():594] send defer +2025-01-03 23:04:08,614 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,614 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 23:04:08,614 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 1 +2025-01-03 23:04:08,614 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,614 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 23:04:08,614 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 23:04:08,614 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 2 +2025-01-03 23:04:08,614 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,614 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 23:04:08,615 INFO HandlerThread:1990615 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 23:04:08,615 DEBUG SystemMonitor:1990615 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 23:04:08,615 DEBUG SystemMonitor:1990615 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 23:04:08,615 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 23:04:08,615 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 23:04:08,670 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 23:04:08,671 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 23:04:08,671 INFO HandlerThread:1990615 [interfaces.py:finish():200] Joined network monitor +2025-01-03 23:04:08,671 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,671 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 23:04:08,671 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 3 +2025-01-03 23:04:08,671 DEBUG SenderThread:1990615 [sender.py:send():379] send: stats +2025-01-03 23:04:08,671 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,672 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 23:04:08,672 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:08,672 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 23:04:08,672 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 4 +2025-01-03 23:04:08,672 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:08,672 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 23:04:08,857 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/mled8cx1-media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:09,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:10,360 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:15,133 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:15,133 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:15,133 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 23:04:15,134 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,134 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 23:04:15,134 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 5 +2025-01-03 23:04:15,134 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:15,135 DEBUG SenderThread:1990615 [sender.py:send():379] send: history +2025-01-03 23:04:15,135 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:04:15,135 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:15,135 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:15,136 DEBUG SenderThread:1990615 [sender.py:send():379] send: summary +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:04:15,136 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 23:04:15,136 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 6 +2025-01-03 23:04:15,137 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:15,137 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 23:04:15,137 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,137 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 23:04:15,140 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:15,255 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 7 +2025-01-03 23:04:15,255 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:15,255 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 23:04:15,255 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:15,255 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 23:04:15,362 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:15,363 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml +2025-01-03 23:04:15,610 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:17,075 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 8 +2025-01-03 23:04:17,075 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:17,076 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:17,076 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 23:04:17,076 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:17,076 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 23:04:17,076 INFO SenderThread:1990615 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 23:04:17,078 INFO SenderThread:1990615 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 23:04:17,080 INFO SenderThread:1990615 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 23:04:17,095 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 9 +2025-01-03 23:04:17,095 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:17,095 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 23:04:17,096 DEBUG SenderThread:1990615 [sender.py:send():379] send: artifact +2025-01-03 23:04:17,363 INFO Thread-12 :1990615 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:17,427 INFO SenderThread:1990615 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'versionIndex': 414}}} +2025-01-03 23:04:17,427 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:17,428 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 23:04:17,428 INFO SenderThread:1990615 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 23:04:17,611 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml config.yaml +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json wandb-summary.json +2025-01-03 23:04:18,364 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt requirements.txt +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-metadata.json wandb-metadata.json +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log output.log +2025-01-03 23:04:18,365 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/diff.patch diff.patch +2025-01-03 23:04:18,367 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 media/videos/eval/4_eval_4_6398d8b01df3925cc7cf.mp4 +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 media/videos/eval/2_eval_2_352b7f510e51bce38993.mp4 +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 media/videos/eval/3_eval_3_0172777c44cdec6ec773.mp4 +2025-01-03 23:04:18,371 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 media/videos/eval/1_eval_1_d3ea4408164b8a140674.mp4 +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 media/videos/eval/0_eval_0_5e4793e12534ec85ec65.mp4 +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json media/table/eval/total_success_rate_ci_6_b44f9defe3679a919f05.table.json +2025-01-03 23:04:18,373 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 10 +2025-01-03 23:04:18,375 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:18,375 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:18,375 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 23:04:18,375 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:18,375 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 23:04:18,375 INFO SenderThread:1990615 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:04:18,556 INFO wandb-upload_1:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/wandb-summary.json +2025-01-03 23:04:18,581 INFO wandb-upload_2:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/requirements.txt +2025-01-03 23:04:18,600 INFO wandb-upload_0:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/config.yaml +2025-01-03 23:04:18,612 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:18,612 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:18,620 INFO wandb-upload_4:1990615 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/files/output.log +2025-01-03 23:04:18,662 INFO wandb-upload_3:1990615 [upload_job.py:push():130] Uploaded file /tmp/tmprs6y3f16wandb/ac16q5jn-events.out.tfevents.1735963111.leonmkim-ROG-Strix-G15CS-G15CS.1990442.0 +2025-01-03 23:04:18,863 INFO Thread-11 (_thread_body):1990615 [sender.py:transition_state():614] send defer: 11 +2025-01-03 23:04:18,863 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:18,863 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 23:04:18,863 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:18,863 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 23:04:18,863 INFO SenderThread:1990615 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 23:04:18,865 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 12 +2025-01-03 23:04:18,865 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:18,865 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 23:04:18,865 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:18,865 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 23:04:18,865 INFO SenderThread:1990615 [file_stream.py:finish():601] file stream finish called +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [file_stream.py:finish():605] file stream finish is done +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 13 +2025-01-03 23:04:19,060 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:19,060 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 23:04:19,060 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 23:04:19,060 INFO SenderThread:1990615 [sender.py:transition_state():614] send defer: 14 +2025-01-03 23:04:19,060 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:04:19,061 INFO HandlerThread:1990615 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 23:04:19,061 DEBUG SenderThread:1990615 [sender.py:send():379] send: final +2025-01-03 23:04:19,061 DEBUG SenderThread:1990615 [sender.py:send():379] send: footer +2025-01-03 23:04:19,061 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: defer +2025-01-03 23:04:19,061 INFO SenderThread:1990615 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 23:04:19,062 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:19,062 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:19,062 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 23:04:19,062 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: server_info +2025-01-03 23:04:19,064 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:04:19,064 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 23:04:19,064 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 23:04:19,065 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:19,126 DEBUG SenderThread:1990615 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:04:19,127 INFO MainThread:1990615 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 23:04:19,127 INFO MainThread:1990615 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 23:04:19,128 INFO MainThread:1990615 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 23:04:19,128 DEBUG HandlerThread:1990615 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 23:04:19,128 INFO HandlerThread:1990615 [handler.py:finish():882] shutting down handler +2025-01-03 23:04:20,065 INFO WriterThread:1990615 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/run-p96n5ha2.wandb +2025-01-03 23:04:20,127 INFO SenderThread:1990615 [sender.py:finish():1615] shutting down sender +2025-01-03 23:04:20,127 INFO SenderThread:1990615 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:04:20,127 INFO SenderThread:1990615 [file_pusher.py:join():175] waiting for file pusher diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug.log b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..610caa6894eea59401dec5141f1286b8c296774f --- /dev/null +++ b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Configure stats pid to 1990442 +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/settings +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:58:30,986 INFO MainThread:1990442 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug.log +2025-01-03 22:58:30,987 INFO MainThread:1990442 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/logs/debug-internal.log +2025-01-03 22:58:30,987 INFO MainThread:1990442 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:58:30,987 INFO MainThread:1990442 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2'} +2025-01-03 22:58:30,988 INFO MainThread:1990442 [wandb_init.py:init():619] starting backend +2025-01-03 22:58:30,988 INFO MainThread:1990442 [wandb_init.py:init():623] setting up manager +2025-01-03 22:58:30,992 INFO MainThread:1990442 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:58:30,993 INFO MainThread:1990442 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:58:31,005 INFO MainThread:1990442 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:58:31,012 INFO MainThread:1990442 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:58:31,263 INFO MainThread:1990442 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:58:31,303 INFO MainThread:1990442 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:58:31,303 INFO MainThread:1990442 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:58:31,636 INFO MainThread:1990442 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:58:31,637 INFO MainThread:1990442 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:58:31,638 INFO MainThread:1990442 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/p96n5ha2/tb, True +2025-01-03 22:58:36,604 INFO MainThread:1990442 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 23:04:20,129 WARNING MsgRouterThr:1990442 [router.py:message_loop():77] message_loop has been closed diff --git a/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/run-p96n5ha2.wandb b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/run-p96n5ha2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0bf6bf0ed1abe7bb4c9a85e5e629b587be1e75ac Binary files /dev/null and b/p96n5ha2/wandb/run-20250103_225830-p96n5ha2/run-p96n5ha2.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..33f839c2877e8a42dccec07fcaf7b6827ca3f9d8 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5baf2eafe24c2835986005b8709cd7e4ef007fbd20bd8e5220647686064a1bf4 +size 910446022 diff --git a/snapshot_10749.pt b/snapshot_10749.pt new file mode 100644 index 0000000000000000000000000000000000000000..19c5f917b1d1ad1d7cb207d7f23429123fcc572e --- /dev/null +++ b/snapshot_10749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24e58db0e39bc1f3bc924cf9dc8ca5ae378033446929f82fd238755dece724e4 +size 910446022 diff --git a/snapshot_12249.pt b/snapshot_12249.pt new file mode 100644 index 0000000000000000000000000000000000000000..09d6393d53750a0b8c6e060b9650e549d35a9d60 --- /dev/null +++ b/snapshot_12249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0526a6700a9076cc5944ea94051a67417a993525e6cc4f2f5b80ded5444cbbb2 +size 910446022 diff --git a/snapshot_12749.pt b/snapshot_12749.pt new file mode 100644 index 0000000000000000000000000000000000000000..13fc3120bbb9fbcdef1f922f58a538b9bf1fdd9b --- /dev/null +++ b/snapshot_12749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33ade9f80a6667af1e974a65ab09a077e88557a9b4d08a9cc3305cd27df72f11 +size 910446022 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7ea257eabdb16431eb472d576afd5b5a8c1a02a --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72556b035f7cd5c693c1788df9dcfa970f95037d54cd16d4217e4c519f2b5293 +size 910446022 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b12a9646e99d9f087cc3db6f784e13be392cb0a --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f2eb076e5b2596c37036dacd9e67bdcef36e8745eb89756579f69d8d3305b8 +size 910445512 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..81fc6a887e12b4fcd711e6a96488baebc1d110da --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d25c9d45083bab4d4889fb44da3a04ab1f3291103536d5a06d2329c38c14425b +size 910446022 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..87bb196a3a1fe601da9331e71e7ff5e9d6ab718d --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1bd4e096cf261faddfdfe6fc2baad35a96d292718ea30a0c463dc57c9702228 +size 910446022 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ae0f224a766bae268390693265db01b06c941df --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8123fafca81e1b4be6193df02555c664df321ee5e8a999bd921123a08613f2a4 +size 910446022 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ee37f4f2e258620ea8d530dcd6b02c72c557d3c --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01ba65f056726be2eff57b8826d32d56105fa958f85ab5b7be2784c2f3da59e2 +size 910446022 diff --git a/snapshot_6749.pt b/snapshot_6749.pt new file mode 100644 index 0000000000000000000000000000000000000000..84a68d5ca1159d9cd72771ea451760b6dcc29da9 --- /dev/null +++ b/snapshot_6749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa71988fc994463dae16673e59e5f0e5b022545ea665709bcada760086507095 +size 910446022 diff --git a/snapshot_6881.pt b/snapshot_6881.pt new file mode 100644 index 0000000000000000000000000000000000000000..84a68d5ca1159d9cd72771ea451760b6dcc29da9 --- /dev/null +++ b/snapshot_6881.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa71988fc994463dae16673e59e5f0e5b022545ea665709bcada760086507095 +size 910446022 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fc0da0fca791391276abd6d61a2f74cd020cc5c --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a59b9b3d75f2aa2a72c94ad554c808b27a026b94230acecf5ec1db397c03c26f +size 910446022 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..88e957e5e5cb99d91f0a13389a7b870cfc3c3564 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2ddfdc601cbe90d8a10cc5adc015e4495936a8d865f0438df77cffc5e896b9 +size 910446022 diff --git a/snapshot_9255.pt b/snapshot_9255.pt new file mode 100644 index 0000000000000000000000000000000000000000..88e957e5e5cb99d91f0a13389a7b870cfc3c3564 --- /dev/null +++ b/snapshot_9255.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2ddfdc601cbe90d8a10cc5adc015e4495936a8d865f0438df77cffc5e896b9 +size 910446022 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..81fc6a887e12b4fcd711e6a96488baebc1d110da --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d25c9d45083bab4d4889fb44da3a04ab1f3291103536d5a06d2329c38c14425b +size 910446022 diff --git a/y2m69p6z/.hydra/config.yaml b/y2m69p6z/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/y2m69p6z/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/y2m69p6z/.hydra/hydra.yaml b/y2m69p6z/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c5f16ad5920ca8a7139ef7464c3ba151182c105 --- /dev/null +++ b/y2m69p6z/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/y2m69p6z/.hydra/overrides.yaml b/y2m69p6z/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/y2m69p6z/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/y2m69p6z/episode_rosbags/aligned_depth_to_color_K.npy b/y2m69p6z/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/y2m69p6z/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/y2m69p6z/episode_rosbags/cam_tf_world.npy b/y2m69p6z/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/y2m69p6z/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/y2m69p6z/episode_rosbags/color_K.npy b/y2m69p6z/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/y2m69p6z/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/y2m69p6z/episode_rosbags/depth_K.npy b/y2m69p6z/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/y2m69p6z/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/y2m69p6z/episode_rosbags/episode_1_2025-01-04-20-44-19.bag b/y2m69p6z/episode_rosbags/episode_1_2025-01-04-20-44-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..6c71e7e38400d179050926736fbbab7a8a9e3ac1 --- /dev/null +++ b/y2m69p6z/episode_rosbags/episode_1_2025-01-04-20-44-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44251adb9202786df6abcdcf0cb5434c8942c8164e5bfcdb2419c82a6bb51170 +size 1444983032 diff --git a/y2m69p6z/eval_robot.log b/y2m69p6z/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..e2c743544315d9e186149b17e17048ed48877e58 --- /dev/null +++ b/y2m69p6z/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 20:42:51,294][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:42:51,297][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/y2m69p6z/eval_video/1_eval.mp4 b/y2m69p6z/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e1a3dd254229297d77b8c379ac749e1a09ea0a3 --- /dev/null +++ b/y2m69p6z/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd84dcdc6f80f9d2f9737cd5b128fffad89b0679db21682937436d95631b0fb +size 872643 diff --git a/y2m69p6z/eval_video/2_eval.mp4 b/y2m69p6z/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d768f86f7f43438bf0419baaeadea9a77768b54 --- /dev/null +++ b/y2m69p6z/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0084bdfe26b3780f6bfb1a7f952057848047fcaf87e4644469d635a431108124 +size 1333372 diff --git a/y2m69p6z/eval_video/3_eval.mp4 b/y2m69p6z/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4539da296a84a030345e63b8ecd311f49c856408 --- /dev/null +++ b/y2m69p6z/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:306a5f59512b8e5f211d238ec11feaff74b42443bfaf86807163c7ce6a2ca386 +size 1394203 diff --git a/y2m69p6z/eval_video/4_eval.mp4 b/y2m69p6z/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6d1255d17f587f1a083477a0c7b90ea6c02c5b9 --- /dev/null +++ b/y2m69p6z/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20e8a1be33290d88548c7d60cccb91697bb3ea26d601cb54de610c8a1f287ee +size 1074570 diff --git a/y2m69p6z/tb/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 b/y2m69p6z/tb/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 new file mode 100644 index 0000000000000000000000000000000000000000..c831d0895b0bc8ab3f01f51346a3b09fc9256926 --- /dev/null +++ b/y2m69p6z/tb/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:287a0badae198a44846d194aa9a84b8b9ecccc988f9fd76224cef2dc79c0d980 +size 6539 diff --git a/y2m69p6z/wandb/debug-internal.log b/y2m69p6z/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..934dc711b017701d049cdd0bac50abd150cf63c7 --- /dev/null +++ b/y2m69p6z/wandb/debug-internal.log @@ -0,0 +1,836 @@ +2025-01-04 20:42:57,170 INFO StreamThr :2015244 [internal.py:wandb_internal():85] W&B internal server running at pid: 2015244, started at: 2025-01-04 20:42:57.170518 +2025-01-04 20:42:57,171 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:42:57,184 INFO WriterThread:2015244 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/run-y2m69p6z.wandb +2025-01-04 20:42:57,184 DEBUG SenderThread:2015244 [sender.py:send():379] send: header +2025-01-04 20:42:57,192 DEBUG SenderThread:2015244 [sender.py:send():379] send: run +2025-01-04 20:42:57,534 INFO SenderThread:2015244 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files +2025-01-04 20:42:57,535 INFO SenderThread:2015244 [sender.py:_start_run_threads():1188] run started: y2m69p6z with start time 1736041377.171353 +2025-01-04 20:42:57,549 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:42:57,549 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:42:57,625 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:42:57,630 DEBUG HandlerThread:2015244 [system_info.py:__init__():26] System info init +2025-01-04 20:42:57,630 DEBUG HandlerThread:2015244 [system_info.py:__init__():41] System info init done +2025-01-04 20:42:57,630 INFO HandlerThread:2015244 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:42:57,630 INFO SystemMonitor:2015244 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:42:57,630 INFO HandlerThread:2015244 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:42:57,631 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:42:57,631 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:42:57,632 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:42:57,632 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:42:57,633 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:42:57,642 DEBUG HandlerThread:2015244 [system_info.py:probe():152] Probing system +2025-01-04 20:42:57,644 DEBUG HandlerThread:2015244 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:42:57,648 DEBUG HandlerThread:2015244 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:42:57,648 DEBUG HandlerThread:2015244 [system_info.py:probe():200] Probing system done +2025-01-04 20:42:57,648 DEBUG HandlerThread:2015244 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:42:57.642913', 'startedAt': '2025-01-05T01:42:57.163246', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.410625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.987, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.982, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 614.3311347961426}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:42:57,649 INFO HandlerThread:2015244 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:42:57,649 INFO HandlerThread:2015244 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:42:57,649 DEBUG HandlerThread:2015244 [system_info.py:_save_code():44] Saving code +2025-01-04 20:42:57,650 DEBUG HandlerThread:2015244 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:42:57,650 DEBUG HandlerThread:2015244 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:42:57,714 DEBUG HandlerThread:2015244 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:42:57,714 INFO HandlerThread:2015244 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:42:57,718 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:42:57,718 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:42:57,718 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:42:57,718 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:42:57,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:42:57,981 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:42:57,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:42:57,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:42:57,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:57,984 INFO HandlerThread:2015244 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/tb" + save: true +} +control { + relay_id: "62daf6bcd4b642bc80d6f18f08d20a32" +} +_info { + stream_id: "y2m69p6z" +} + +2025-01-04 20:42:57,984 DEBUG HandlerThread:2015244 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:42:57,991 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/asm4h0zx-code/FISH/eval_robot.py +2025-01-04 20:42:57,992 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/yivitxgi-diff.patch +2025-01-04 20:42:58,010 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/bor37lpy-wandb-metadata.json +2025-01-04 20:42:58,119 DEBUG SenderThread:2015244 [sender.py:send():379] send: telemetry +2025-01-04 20:42:58,119 DEBUG SenderThread:2015244 [sender.py:send():379] send: tbrecord +2025-01-04 20:42:58,120 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:42:58,120 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 with policy live +2025-01-04 20:42:58,315 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/t9njwcfa-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:42:58,536 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/diff.patch +2025-01-04 20:42:58,536 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-metadata.json +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH/eval_robot.py +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code +2025-01-04 20:42:58,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:59,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:00,536 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:00,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:01,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:02,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:02,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:03,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:04,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:05,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:06,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:07,846 DEBUG SenderThread:2015244 [sender.py:send():379] send: config +2025-01-04 20:43:07,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:07,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:08,538 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:08,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:09,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:10,538 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:10,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:11,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:12,539 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:12,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:12,981 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:12,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:13,131 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:13,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:14,540 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:14,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:15,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:16,540 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:16,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:17,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:18,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:18,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:19,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:20,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:21,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:22,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:23,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:23,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:24,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:25,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:26,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:27,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:27,981 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:27,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:28,544 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:28,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:29,568 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:29,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:30,544 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:30,544 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml +2025-01-04 20:43:30,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:31,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:32,545 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:32,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:33,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:34,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:34,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:35,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:36,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:37,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:38,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:39,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:40,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:40,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:41,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:42,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:42,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:43,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:43,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:44,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:45,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:46,132 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:46,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:47,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:48,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:49,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:50,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:51,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:51,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:52,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:53,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:54,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:55,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:56,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:56,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:57,633 DEBUG SystemMonitor:2015244 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:43:57,634 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:43:57,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:57,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:58,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:58,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:59,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:00,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:01,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:02,127 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:02,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:03,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:04,553 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:04,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:05,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:07,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:07,128 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:07,169 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:44:07,170 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:44:07,170 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 with policy now +2025-01-04 20:44:07,170 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:44:07,170 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:44:07,171 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:44:07,435 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/46xbgykn-media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:44:07,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:44:07,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:44:07,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:44:07,555 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media +2025-01-04 20:44:07,555 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos +2025-01-04 20:44:07,808 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/3voobc0f-media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:44:08,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:08,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:09,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:10,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:11,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:12,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:12,556 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:12,713 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:12,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:12,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:13,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:14,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:14,557 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:15,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:16,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:16,557 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:44:16,803 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/3f88gc4h-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:44:17,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:17,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:18,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:18,558 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:19,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:20,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:20,559 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:21,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:22,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:22,559 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:22,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:23,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:24,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:25,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:26,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:27,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:27,634 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:44:27,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:27,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:28,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:28,110 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:29,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:30,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:31,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:32,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:33,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:33,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:34,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:35,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:36,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:36,564 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:37,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:38,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:38,565 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:38,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:39,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:40,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:40,565 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:40,703 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:44:40,704 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 with policy now +2025-01-04 20:44:40,709 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:44:40,709 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:44:40,710 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:44:40,710 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:44:40,942 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/mk2at2d9-media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:44:41,006 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:41,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:44:41,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:44:41,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:44:41,857 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/bwdik4t6-media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:44:42,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:42,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:42,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:42,987 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:43,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:44,008 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:44,055 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:45,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:46,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:46,567 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:47,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:48,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:48,568 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:49,008 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:49,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:50,008 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:51,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:52,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:53,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:54,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:54,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:55,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:56,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:57,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:57,634 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:44:57,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:57,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:58,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:59,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:00,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:00,104 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:01,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:02,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:03,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:04,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:05,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:05,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:06,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:07,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:08,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:09,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:10,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:10,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:11,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:12,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:12,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:12,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:13,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:14,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:15,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:16,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:16,115 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:17,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:18,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:19,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:20,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:21,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:21,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:22,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:22,579 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:45:22,819 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/sjta8s19-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:45:23,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:23,493 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:45:23,493 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:45:23,494 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:45:23,579 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:45:23,580 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:45:24,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:24,580 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:25,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:26,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:27,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:27,564 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:27,635 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:45:27,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:27,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:28,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:28,581 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:29,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:30,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:31,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:32,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:32,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:33,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:34,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:35,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:36,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:37,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:37,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:38,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:39,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:40,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:41,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:42,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:42,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:42,984 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:43,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:43,096 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:44,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:45,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:46,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:47,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:48,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:48,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:49,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:50,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:51,020 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:52,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:53,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:53,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:54,019 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:55,019 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:56,024 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:56,590 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:57,026 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:57,367 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:45:57,368 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 with policy now +2025-01-04 20:45:57,375 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:45:57,376 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:45:57,376 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:45:57,376 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:45:57,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:45:57,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:45:57,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:45:57,635 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:45:57,642 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/4bmude5n-media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:45:57,937 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/a0d093mt-media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:45:57,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:57,989 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:58,031 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:58,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:59,032 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:59,090 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:59,592 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:00,031 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:01,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:02,031 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:02,593 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:03,032 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:04,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:04,593 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:04,833 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:05,032 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:06,035 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:07,037 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:08,036 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:09,036 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:09,999 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:10,037 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:11,038 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:12,037 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:12,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:12,989 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:13,041 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:14,042 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:15,039 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:15,097 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:16,044 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:17,042 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:18,041 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:19,043 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:19,279 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:46:19,279 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:46:19,295 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:46:19,598 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:46:19,599 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:46:20,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:20,105 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:20,599 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:21,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:22,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:22,599 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:23,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:24,044 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:24,600 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:25,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:25,354 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:26,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:27,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:27,636 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:46:27,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:27,984 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:28,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:29,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:30,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:30,602 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:31,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:31,285 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:32,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:33,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:34,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:35,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:36,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:36,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:37,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:38,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:39,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:40,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:41,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:42,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:42,209 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:42,608 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:42,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:42,984 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:43,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:44,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:45,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:46,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:47,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:47,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:48,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:49,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:50,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:51,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:52,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:53,050 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:53,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:54,050 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:55,055 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:55,628 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:46:55,628 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 with policy now +2025-01-04 20:46:55,629 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:46:55,631 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:46:55,631 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:46:55,632 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:46:55,886 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/gzbi965y-media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:46:56,050 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:56,612 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:46:56,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:56,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:46:56,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:46:56,901 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/tfqyskry-media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:46:57,056 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:57,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:57,636 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:46:57,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:57,989 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:58,057 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:59,056 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:59,076 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:00,058 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:00,614 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:01,056 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:02,054 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:03,062 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:04,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:04,615 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:05,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:05,065 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:06,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:07,059 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:08,059 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:09,061 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:10,060 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:11,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:11,060 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:12,060 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:12,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:12,987 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:13,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:14,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:15,068 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:16,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:16,347 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:47:16,347 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:47:16,348 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:16,348 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:47:16,619 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:47:16,619 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:47:16,619 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:16,862 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/skmguk37-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:47:17,066 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:18,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:18,620 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:19,067 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:20,072 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:20,620 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:21,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:22,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:22,145 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:22,621 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:23,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:24,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:25,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:26,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:27,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:27,637 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:47:27,638 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:27,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:27,986 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:28,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:29,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:30,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:31,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:32,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:33,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:33,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:34,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:35,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:36,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:37,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:38,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:39,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:39,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:40,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:41,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:42,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:42,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:42,986 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:43,072 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:44,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:44,078 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:44,274 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:47:44,274 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 with policy now +2025-01-04 20:47:44,281 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:47:44,282 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:47:44,282 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:47:44,283 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:47:44,558 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/31vblyuf-media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:47:44,886 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/y92u9ang-media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:47:45,084 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:45,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:46,085 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:46,630 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:47,084 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:47,630 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:48,106 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:48,630 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:49,088 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:50,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:50,086 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:51,084 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:52,087 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:52,632 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:53,109 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:54,099 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:55,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:55,111 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:56,111 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:57,102 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:57,637 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:47:57,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:57,994 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:58,107 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:59,105 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:00,107 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:00,143 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:01,110 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:02,111 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:03,124 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:04,115 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:05,121 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:05,420 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:05,421 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:05,421 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:05,421 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:05,447 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:48:05,447 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table +2025-01-04 20:48:05,659 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:48:05,814 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/dxaa2x5b-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:05,952 INFO wandb-upload_0:2015244 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp33fmuk32 +2025-01-04 20:48:06,081 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:48:06,091 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:48:06,112 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:06,364 INFO SenderThread:2015244 [sender.py:send_request_log_artifact():1525] logged artifact run-y2m69p6z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExNjYyNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDg1Ng==', 'latestArtifact': None}} +2025-01-04 20:48:06,364 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:48:06,365 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 20:48:06,365 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:06,365 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,365 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,365 DEBUG SenderThread:2015244 [sender.py:send():379] send: telemetry +2025-01-04 20:48:06,366 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,366 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,366 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,366 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,367 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,367 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,367 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,368 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,368 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,368 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,368 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,370 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,370 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,371 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,371 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,371 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,371 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,372 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,372 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,372 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,372 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,373 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,373 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:48:06,614 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/i2yb21h8-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:06,638 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval +2025-01-04 20:48:06,759 INFO wandb-upload_0:2015244 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdxcpnzhi +2025-01-04 20:48:06,849 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/tgrhi7ko-media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:07,113 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:07,118 INFO SenderThread:2015244 [sender.py:send_request_log_artifact():1525] logged artifact run-y2m69p6z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExNjYzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDg1OQ==', 'latestArtifact': None}} +2025-01-04 20:48:07,118 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:48:07,118 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json with policy now +2025-01-04 20:48:07,118 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:07,118 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:07,119 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:07,362 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/pl3lc1p2-media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:07,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:08,113 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:08,638 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:09,069 DEBUG SenderThread:2015244 [sender.py:send():379] send: exit +2025-01-04 20:48:09,069 INFO SenderThread:2015244 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:send_exit():588] handling runtime: 311 +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:send_exit():594] send defer +2025-01-04 20:48:09,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,070 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:48:09,070 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:48:09,071 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:48:09,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,071 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:48:09,071 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,071 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:48:09,071 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:48:09,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,071 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:48:09,071 INFO HandlerThread:2015244 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:48:09,071 DEBUG SystemMonitor:2015244 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:48:09,071 DEBUG SystemMonitor:2015244 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:48:09,072 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:48:09,072 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:48:09,123 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:48:09,123 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:48:09,124 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:48:09,124 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,124 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:48:09,124 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:48:09,124 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:48:09,124 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,124 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:48:09,125 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,125 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:48:09,125 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:48:09,125 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,125 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:48:09,638 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:10,639 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:15,426 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:15,426 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:15,426 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:48:15,427 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,427 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:48:15,428 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:48:15,428 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:15,428 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:15,428 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:48:15,428 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:15,429 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:15,430 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:15,430 DEBUG SenderThread:2015244 [sender.py:send():379] send: summary +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:15,431 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:48:15,431 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:15,431 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:48:15,431 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:48:15,435 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:15,546 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:48:15,546 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:15,546 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:48:15,546 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,546 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:48:15,641 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:15,641 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml +2025-01-04 20:48:16,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:16,762 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:48:16,762 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:16,763 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:16,763 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:48:16,763 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:16,763 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:48:16,763 INFO SenderThread:2015244 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:48:16,766 INFO SenderThread:2015244 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:48:16,768 INFO SenderThread:2015244 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:48:16,780 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:48:16,780 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:16,780 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:48:16,781 DEBUG SenderThread:2015244 [sender.py:send():379] send: artifact +2025-01-04 20:48:16,991 INFO SenderThread:2015244 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'versionIndex': 417}}} +2025-01-04 20:48:16,992 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:16,992 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:48:16,992 INFO SenderThread:2015244 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:48:17,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:17,642 INFO SenderThread:2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:17,642 INFO SenderThread:2015244 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files +2025-01-04 20:48:17,642 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml config.yaml +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json wandb-summary.json +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt requirements.txt +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:17,646 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log output.log +2025-01-04 20:48:17,646 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/diff.patch diff.patch +2025-01-04 20:48:17,648 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:48:17,651 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:17,652 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:17,652 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:48:17,652 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:17,652 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:48:17,652 INFO SenderThread:2015244 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:48:17,824 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt +2025-01-04 20:48:17,824 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:17,839 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml +2025-01-04 20:48:17,890 INFO wandb-upload_4:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:17,912 INFO wandb-upload_3:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/dvffvggh-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:18,072 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:18,072 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:18,113 INFO Thread-11 (_thread_body):2015244 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:48:18,113 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,113 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:48:18,113 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,113 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:48:18,113 INFO SenderThread:2015244 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:48:18,115 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:48:18,115 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,115 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:48:18,115 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,115 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:48:18,115 INFO SenderThread:2015244 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:48:18,312 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,312 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:48:18,312 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:48:18,312 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send():379] send: final +2025-01-04 20:48:18,313 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send():379] send: footer +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,313 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:48:18,313 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:18,314 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:48:18,314 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:18,314 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:48:18,315 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:48:18,316 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:48:18,316 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:18,362 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:18,362 INFO MainThread:2015244 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:48:18,363 INFO MainThread:2015244 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:48:18,363 INFO MainThread:2015244 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:48:18,363 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:48:18,363 INFO HandlerThread:2015244 [handler.py:finish():882] shutting down handler +2025-01-04 20:48:19,314 INFO WriterThread:2015244 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/run-y2m69p6z.wandb +2025-01-04 20:48:19,362 INFO SenderThread:2015244 [sender.py:finish():1615] shutting down sender +2025-01-04 20:48:19,362 INFO SenderThread:2015244 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:48:19,363 INFO SenderThread:2015244 [file_pusher.py:join():175] waiting for file pusher diff --git a/y2m69p6z/wandb/debug.log b/y2m69p6z/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3ac8c622adb9ed728a458cd0a4cae6ecb524b55f --- /dev/null +++ b/y2m69p6z/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Configure stats pid to 2015050 +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/settings +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug.log +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug-internal.log +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:42:57,166 INFO MainThread:2015050 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z'} +2025-01-04 20:42:57,166 INFO MainThread:2015050 [wandb_init.py:init():619] starting backend +2025-01-04 20:42:57,166 INFO MainThread:2015050 [wandb_init.py:init():623] setting up manager +2025-01-04 20:42:57,169 INFO MainThread:2015050 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:42:57,171 INFO MainThread:2015050 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:42:57,183 INFO MainThread:2015050 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:42:57,190 INFO MainThread:2015050 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:42:57,548 INFO MainThread:2015050 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:42:57,620 INFO MainThread:2015050 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:42:57,620 INFO MainThread:2015050 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:42:57,984 INFO MainThread:2015050 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:42:57,984 INFO MainThread:2015050 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/tb, True +2025-01-04 20:43:07,846 INFO MainThread:2015050 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:48:19,364 WARNING MsgRouterThr:2015050 [router.py:message_loop():77] message_loop has been closed diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH/eval_robot.py b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adfb9a3079cc8c42dd64ca8b0d22203fc3d0f172 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736041377 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/diff.patch b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json new file mode 100644 index 0000000000000000000000000000000000000000..f90e38c3596783d741f259104ec84831368043f4 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[16, 40, 0.4, 0.24612817173313217, 0.5785249471507379, 0.25338669871255737, 0.5486178100011865, 0.2711888030629016, 0.5326476693088086, 0.3148406781626518, 0.5073943734154217]]} \ No newline at end of file diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1607e6623b8d94938d04c4c004d34505152a6dea --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4268b13f1b81246fc5ef8324c29a2330d3937259bbffbdcde004512ec893fb5c +size 1520919 diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e1a3dd254229297d77b8c379ac749e1a09ea0a3 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd84dcdc6f80f9d2f9737cd5b128fffad89b0679db21682937436d95631b0fb +size 872643 diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d768f86f7f43438bf0419baaeadea9a77768b54 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0084bdfe26b3780f6bfb1a7f952057848047fcaf87e4644469d635a431108124 +size 1333372 diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4539da296a84a030345e63b8ecd311f49c856408 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:306a5f59512b8e5f211d238ec11feaff74b42443bfaf86807163c7ce6a2ca386 +size 1394203 diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6d1255d17f587f1a083477a0c7b90ea6c02c5b9 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20e8a1be33290d88548c7d60cccb91697bb3ea26d601cb54de610c8a1f287ee +size 1074570 diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ac6b0309d42b5975d2d504c0371681a6345bd467 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log @@ -0,0 +1,297 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736041387.846951]: resetting environment +[INFO] [1736041387.852045]: cleared current plan +[INFO] [1736041387.853054]: moving to home +[INFO] [1736041388.855073]: reached home +[INFO] [1736041388.856554]: reset action history +[INFO] [1736041391.567326]: environment reset +Starting episode 0 +[INFO] [1736041391.572615]: resetting environment +[INFO] [1736041391.578864]: cleared current plan +[INFO] [1736041391.579730]: moving to home +[INFO] [1736041392.581281]: reached home +[INFO] [1736041392.582771]: reset action history +[INFO] [1736041395.292441]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736041407.510156]: resetting environment +[INFO] [1736041407.514011]: cleared current plan +[INFO] [1736041407.515204]: moving to home +[INFO] [1736041408.517268]: reached home +[INFO] [1736041408.518580]: reset action history +[INFO] [1736041411.277238]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/episode_rosbags/episode_0_2025-01-04-20-43-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2015050.log +started roslaunch server http://158.130.50.37:34407/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2015759] +process[print_text-2]: started with pid [2015783] +process[rosbag_record-3]: started with pid [2015784] +started bagging! +For topic gripper_width: timestamp difference is 107459453 for nearest: 1736041443618268636 - target: 1736041443510809183 at idx 249 +[INFO] [1736041446.200472]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736041446.926568]: Stored episode 1. +[INFO] [1736041446.926839]: Saving video... +[INFO] [1736041447.127514]: Video saved! +Starting episode 1 +[INFO] [1736041447.147152]: resetting environment +[INFO] [1736041447.157910]: cleared current plan +[INFO] [1736041447.162550]: moving to home +[INFO] [1736041450.669217]: reached home +[INFO] [1736041450.669669]: reset action history +[INFO] [1736041453.383837]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041455.945348]: resetting environment +[INFO] [1736041455.952007]: cleared current plan +[INFO] [1736041455.952210]: moving to home +[INFO] [1736041456.953270]: reached home +[INFO] [1736041456.953919]: reset action history +[INFO] [1736041459.662334]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/episode_rosbags/episode_1_2025-01-04-20-44-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2015050.log +started roslaunch server http://158.130.50.37:39045/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2015990] +process[print_text-5]: started with pid [2015991] +process[rosbag_record-6]: started with pid [2016015] +started bagging! +For topic gripper_width: timestamp difference is -100906391 for nearest: 1736041474351577978 - target: 1736041474452484369 at idx 188 +For topic gripper_width: timestamp difference is -232441552 for nearest: 1736041476551543348 - target: 1736041476783984900 at idx 220 +preempting episode +[INFO] [1736041479.193688]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736041479.967051]: Stored episode 2. +[INFO] [1736041479.967328]: Saving video... +[INFO] [1736041480.672876]: Video saved! +Starting episode 2 +[INFO] [1736041480.720866]: resetting environment +[INFO] [1736041480.740428]: cleared current plan +[INFO] [1736041480.740805]: moving to home +[WARN] [1736041480.549422]: Plan exhausted +[WARN] [1736041480.589398]: Plan exhausted +[WARN] [1736041480.631329]: Plan exhausted +[WARN] [1736041480.670964]: Plan exhausted +[WARN] [1736041480.709461]: Plan exhausted +[INFO] [1736041484.243917]: reached home +[INFO] [1736041484.246288]: reset action history +[INFO] [1736041487.017278]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041522.580847]: resetting environment +[INFO] [1736041522.591173]: cleared current plan +[INFO] [1736041522.591363]: moving to home +[INFO] [1736041523.593532]: reached home +[INFO] [1736041523.595779]: reset action history +[INFO] [1736041526.306473]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/episode_rosbags/episode_2_2025-01-04-20-45-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2015050.log +started roslaunch server http://158.130.50.37:37615/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2016235] +process[print_text-8]: started with pid [2016236] +process[rosbag_record-9]: started with pid [2016260] +started bagging! +preempting episode +[INFO] [1736041555.650809]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736041556.616123]: Stored episode 3. +[INFO] [1736041556.618309]: Saving video... +[INFO] [1736041557.335068]: Video saved! +Starting episode 3 +[INFO] [1736041557.380777]: resetting environment +[INFO] [1736041557.398004]: cleared current plan +[INFO] [1736041557.399668]: moving to home +[WARN] [1736041556.847614]: Plan exhausted +[WARN] [1736041556.887626]: Plan exhausted +[WARN] [1736041556.928212]: Plan exhausted +[WARN] [1736041556.968641]: Plan exhausted +[WARN] [1736041557.007652]: Plan exhausted +[WARN] [1736041557.048374]: Plan exhausted +[WARN] [1736041557.089363]: Plan exhausted +[WARN] [1736041557.128929]: Plan exhausted +[WARN] [1736041557.167615]: Plan exhausted +[WARN] [1736041557.207616]: Plan exhausted +[WARN] [1736041557.248588]: Plan exhausted +[WARN] [1736041557.291113]: Plan exhausted +[WARN] [1736041557.327951]: Plan exhausted +[WARN] [1736041557.368483]: Plan exhausted +[INFO] [1736041561.005394]: reached home +[INFO] [1736041561.010118]: reset action history +[INFO] [1736041563.790241]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041579.018416]: resetting environment +[INFO] [1736041579.046277]: cleared current plan +[INFO] [1736041579.050216]: moving to home +[INFO] [1736041580.055910]: reached home +[INFO] [1736041580.059396]: reset action history +[INFO] [1736041582.857253]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/episode_rosbags/episode_3_2025-01-04-20-46-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2015050.log +started roslaunch server http://158.130.50.37:39131/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2016789] +process[print_text-11]: started with pid [2016790] +process[rosbag_record-12]: started with pid [2016791] +started bagging! +For topic gripper_width: timestamp difference is -109114635 for nearest: 1736041588051573765 - target: 1736041588160688400 at idx 73 +For topic gripper_width: timestamp difference is 100481978 for nearest: 1736041600151576272 - target: 1736041600051094294 at idx 248 +preempting episode +[INFO] [1736041614.354606]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736041615.131965]: Stored episode 4. +[INFO] [1736041615.133216]: Saving video... +[INFO] [1736041615.594462]: Video saved! +Starting episode 4 +[INFO] [1736041615.644775]: resetting environment +[INFO] [1736041615.669595]: cleared current plan +[INFO] [1736041615.669934]: moving to home +[WARN] [1736041615.567835]: Plan exhausted +[WARN] [1736041615.609323]: Plan exhausted +[WARN] [1736041615.648223]: Plan exhausted +[INFO] [1736041619.075666]: reached home +[INFO] [1736041619.076607]: reset action history +[INFO] [1736041621.918871]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041635.876514]: resetting environment +[INFO] [1736041635.904320]: cleared current plan +[INFO] [1736041635.906978]: moving to home +[INFO] [1736041636.919176]: reached home +[INFO] [1736041636.923680]: reset action history +[INFO] [1736041639.708718]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/episode_rosbags/episode_4_2025-01-04-20-47-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2015050.log +started roslaunch server http://158.130.50.37:37477/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2017013] +process[print_text-14]: started with pid [2017014] +process[rosbag_record-15]: started with pid [2017015] +started bagging! +preempting episode +[INFO] [1736041662.935621]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1736041663.712903]: Stored episode 5. +[INFO] [1736041663.714195]: Saving video... +[WARN] [1736041664.008400]: Plan exhausted +[WARN] [1736041664.048061]: Plan exhausted +[WARN] [1736041664.087648]: Plan exhausted +[WARN] [1736041664.128354]: Plan exhausted +[WARN] [1736041664.167820]: Plan exhausted +[WARN] [1736041664.209474]: Plan exhausted +[INFO] [1736041664.216988]: Video saved! +[INFO] [1736041664.285578]: resetting environment +[INFO] [1736041664.300019]: cleared current plan +[INFO] [1736041664.300735]: moving to home +[WARN] [1736041664.247817]: Plan exhausted +[WARN] [1736041664.294084]: Plan exhausted +[INFO] [1736041667.803007]: reached home +[INFO] [1736041667.807658]: reset action history +[INFO] [1736041670.614601]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-metadata.json b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a4b71b5867e725c56e390f2a9ff0dfbd73d482cc --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T01:42:57.642913", + "startedAt": "2025-01-05T01:42:57.163246", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.410625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.987, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.982, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 614.3311347961426 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7760f963aefc3d8668d70b7d711beaf87c510b5f --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "4268b13f1b81246fc5ef8324c29a2330d3937259bbffbdcde004512ec893fb5c", "size": 1520919, "path": "media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4"}, "global_step": 520, "_timestamp": 1736041685.983278, "_runtime": 308.8119249343872, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "4fd84dcdc6f80f9d2f9737cd5b128fffad89b0679db21682937436d95631b0fb", "size": 872643, "path": "media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4"}, "eval/num_success": 0.0, "episode": 40.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "0084bdfe26b3780f6bfb1a7f952057848047fcaf87e4644469d635a431108124", "size": 1333372, "path": "media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "306a5f59512b8e5f211d238ec11feaff74b42443bfaf86807163c7ce6a2ca386", "size": 1394203, "path": "media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "b20e8a1be33290d88548c7d60cccb91697bb3ea26d601cb54de610c8a1f287ee", "size": 1074570, "path": "media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://cw30ju8imw9odwdlgljby61y8qdh2i76lxn3aghga51nvk07aqaeelj9izk339yf9olh28wz3jg6wpvurty10e7na9hpay6oj4oz9wny4fmydo4dhjg46imgmrkki26r/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://pumfmi6kok9cz5zxrh6ob1q0owqlzuq4a1dns34tlgwogx7npv7mhtrf4miuv7nqoqqlb7fwjaoboijqnz638pnw3vkk0jdvs4b91xxc1a4orc7x3hnwpeo49p9xcjxz:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 16, "total_num_episodes": 40, "total_success_rate": 0.4, "total_success_rate_lb_0.01": 0.24612817173313217, "total_success_rate_ub_0.01": 0.5785249471507379, "total_success_rate_lb_0.025": 0.25338669871255737, "total_success_rate_ub_0.025": 0.5486178100011865, "total_success_rate_lb_0.05": 0.2711888030629016, "total_success_rate_ub_0.05": 0.5326476693088086, "total_success_rate_lb_0.1": 0.3148406781626518, "total_success_rate_ub_0.1": 0.5073943734154217, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "12715ecfba84d39fa53953a0a7307a0534a4a28569349fdc89a9ce0d340cca04", "size": 507, "artifact_path": "wandb-client-artifact://lk2pnnnlu1zdi7khvx1bhw34gr9xcx684qtile51b91gfrbjtxz2c79mnvmffau2iscrk15s6ofslgay8sdnko5wwumf2vvruqnhtkv0bm67wrntsend6zytxbjx6hya/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://fqi9x7cbmegj9tv1760jknhpztpkvyeb37bsga4y5hfnxm8n9vf5y3m1oxaqclj6l5ep3qsoicm6hw1bzjl4zzo1al9jhu3x4ymd1x0ip65pxl263j4htr75yfbwmxq4:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 311}, "eval/total_success_rate": 0.4000000059604645, "eval/total_success_rate_lb_0.01": 0.2461281716823578, "eval/total_success_rate_ub_0.01": 0.5785249471664429, "eval/total_success_rate_lb_0.025": 0.2533867061138153, "eval/total_success_rate_ub_0.025": 0.5486177802085876, "eval/total_success_rate_lb_0.05": 0.27118879556655884, "eval/total_success_rate_ub_0.05": 0.5326476693153381, "eval/total_success_rate_lb_0.1": 0.3148406744003296, "eval/total_success_rate_ub_0.1": 0.5073943734169006} \ No newline at end of file diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug-internal.log b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..934dc711b017701d049cdd0bac50abd150cf63c7 --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug-internal.log @@ -0,0 +1,836 @@ +2025-01-04 20:42:57,170 INFO StreamThr :2015244 [internal.py:wandb_internal():85] W&B internal server running at pid: 2015244, started at: 2025-01-04 20:42:57.170518 +2025-01-04 20:42:57,171 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:42:57,184 INFO WriterThread:2015244 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/run-y2m69p6z.wandb +2025-01-04 20:42:57,184 DEBUG SenderThread:2015244 [sender.py:send():379] send: header +2025-01-04 20:42:57,192 DEBUG SenderThread:2015244 [sender.py:send():379] send: run +2025-01-04 20:42:57,534 INFO SenderThread:2015244 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files +2025-01-04 20:42:57,535 INFO SenderThread:2015244 [sender.py:_start_run_threads():1188] run started: y2m69p6z with start time 1736041377.171353 +2025-01-04 20:42:57,549 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:42:57,549 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:42:57,625 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:42:57,630 DEBUG HandlerThread:2015244 [system_info.py:__init__():26] System info init +2025-01-04 20:42:57,630 DEBUG HandlerThread:2015244 [system_info.py:__init__():41] System info init done +2025-01-04 20:42:57,630 INFO HandlerThread:2015244 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:42:57,630 INFO SystemMonitor:2015244 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:42:57,630 INFO HandlerThread:2015244 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:42:57,631 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:42:57,631 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:42:57,632 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:42:57,632 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:42:57,633 INFO SystemMonitor:2015244 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:42:57,642 DEBUG HandlerThread:2015244 [system_info.py:probe():152] Probing system +2025-01-04 20:42:57,644 DEBUG HandlerThread:2015244 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:42:57,648 DEBUG HandlerThread:2015244 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:42:57,648 DEBUG HandlerThread:2015244 [system_info.py:probe():200] Probing system done +2025-01-04 20:42:57,648 DEBUG HandlerThread:2015244 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:42:57.642913', 'startedAt': '2025-01-05T01:42:57.163246', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.410625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.987, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.982, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 614.3311347961426}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:42:57,649 INFO HandlerThread:2015244 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:42:57,649 INFO HandlerThread:2015244 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:42:57,649 DEBUG HandlerThread:2015244 [system_info.py:_save_code():44] Saving code +2025-01-04 20:42:57,650 DEBUG HandlerThread:2015244 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:42:57,650 DEBUG HandlerThread:2015244 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:42:57,714 DEBUG HandlerThread:2015244 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:42:57,714 INFO HandlerThread:2015244 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:42:57,718 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:42:57,718 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:42:57,718 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:42:57,718 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:42:57,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:42:57,981 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:42:57,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:42:57,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:42:57,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:57,984 INFO HandlerThread:2015244 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/tb" + save: true +} +control { + relay_id: "62daf6bcd4b642bc80d6f18f08d20a32" +} +_info { + stream_id: "y2m69p6z" +} + +2025-01-04 20:42:57,984 DEBUG HandlerThread:2015244 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:42:57,991 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/asm4h0zx-code/FISH/eval_robot.py +2025-01-04 20:42:57,992 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/yivitxgi-diff.patch +2025-01-04 20:42:58,010 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/bor37lpy-wandb-metadata.json +2025-01-04 20:42:58,119 DEBUG SenderThread:2015244 [sender.py:send():379] send: telemetry +2025-01-04 20:42:58,119 DEBUG SenderThread:2015244 [sender.py:send():379] send: tbrecord +2025-01-04 20:42:58,120 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:42:58,120 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 with policy live +2025-01-04 20:42:58,315 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/t9njwcfa-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:42:58,536 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/diff.patch +2025-01-04 20:42:58,536 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-metadata.json +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH/eval_robot.py +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH +2025-01-04 20:42:58,537 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code +2025-01-04 20:42:58,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:59,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:00,536 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:00,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:01,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:02,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:02,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:03,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:04,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:05,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:06,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:07,846 DEBUG SenderThread:2015244 [sender.py:send():379] send: config +2025-01-04 20:43:07,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:07,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:08,538 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:08,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:09,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:10,538 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:10,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:11,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:12,539 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:12,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:12,981 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:12,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:13,131 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:13,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:14,540 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:14,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:15,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:16,540 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:16,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:17,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:18,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:18,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:19,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:20,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:21,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:22,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:23,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:23,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:24,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:25,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:26,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:27,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:27,981 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:27,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:28,544 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:28,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:29,568 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:29,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:30,544 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:30,544 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml +2025-01-04 20:43:30,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:31,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:32,545 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:43:32,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:33,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:34,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:34,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:35,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:36,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:37,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:38,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:39,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:40,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:40,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:41,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:42,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:42,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:43,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:43,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:44,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:45,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:46,132 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:46,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:47,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:48,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:49,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:50,991 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:51,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:51,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:52,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:53,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:54,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:55,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:56,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:43:56,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:57,633 DEBUG SystemMonitor:2015244 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:43:57,634 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:43:57,981 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:43:57,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:43:58,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:58,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:43:59,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:00,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:01,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:02,127 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:02,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:03,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:04,553 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:04,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:05,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:07,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:07,128 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:07,169 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:44:07,170 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:44:07,170 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 with policy now +2025-01-04 20:44:07,170 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:44:07,170 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:44:07,171 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:44:07,435 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/46xbgykn-media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:44:07,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:44:07,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:44:07,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:44:07,555 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media +2025-01-04 20:44:07,555 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos +2025-01-04 20:44:07,808 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/3voobc0f-media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:44:08,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:08,554 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:09,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:10,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:11,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:12,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:12,556 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:12,713 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:12,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:12,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:13,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:14,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:14,557 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:15,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:16,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:16,557 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:44:16,803 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/3f88gc4h-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:44:17,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:17,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:18,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:18,558 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:19,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:20,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:20,559 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:21,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:22,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:22,559 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:22,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:23,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:24,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:25,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:26,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:27,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:27,634 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:44:27,982 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:27,982 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:28,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:28,110 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:29,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:30,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:31,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:32,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:33,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:33,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:34,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:35,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:36,004 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:36,564 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:37,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:38,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:38,565 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:38,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:39,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:40,005 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:40,565 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:40,703 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:44:40,704 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 with policy now +2025-01-04 20:44:40,709 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:44:40,709 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:44:40,710 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:44:40,710 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:44:40,942 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/mk2at2d9-media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:44:41,006 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:41,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:44:41,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:44:41,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:44:41,857 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/bwdik4t6-media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:44:42,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:42,566 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:42,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:42,987 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:43,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:44,008 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:44,055 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:45,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:46,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:46,567 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:47,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:48,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:48,568 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:44:49,008 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:49,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:50,008 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:51,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:52,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:53,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:54,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:54,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:44:55,009 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:56,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:57,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:57,634 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:44:57,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:44:57,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:44:58,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:44:59,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:00,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:00,104 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:01,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:02,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:03,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:04,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:05,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:05,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:06,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:07,011 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:08,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:09,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:10,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:10,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:11,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:12,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:12,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:12,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:13,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:14,012 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:15,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:16,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:16,115 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:17,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:18,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:19,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:20,013 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:21,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:21,990 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:22,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:22,579 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:45:22,819 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/sjta8s19-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:45:23,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:23,493 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:45:23,493 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:45:23,494 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:45:23,579 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:45:23,580 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:45:24,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:24,580 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:25,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:26,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:27,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:27,564 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:27,635 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:45:27,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:27,983 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:28,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:28,581 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:29,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:30,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:31,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:32,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:32,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:33,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:34,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:35,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:36,015 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:37,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:37,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:38,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:39,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:40,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:41,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:42,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:42,983 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:42,984 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:43,025 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:43,096 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:44,017 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:45,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:46,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:47,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:48,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:48,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:49,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:50,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:51,020 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:52,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:53,018 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:53,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:54,019 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:55,019 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:56,024 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:56,590 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:57,026 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:57,367 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:45:57,368 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 with policy now +2025-01-04 20:45:57,375 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:45:57,376 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:45:57,376 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:45:57,376 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:45:57,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:45:57,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:45:57,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:45:57,635 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:45:57,642 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/4bmude5n-media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:45:57,937 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/a0d093mt-media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:45:57,988 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:45:57,989 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:45:58,031 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:58,591 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:45:59,032 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:45:59,090 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:45:59,592 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:00,031 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:01,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:02,031 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:02,593 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:03,032 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:04,029 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:04,593 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:04,833 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:05,032 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:06,035 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:07,037 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:08,036 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:09,036 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:09,999 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:10,037 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:11,038 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:12,037 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:12,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:12,989 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:13,041 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:14,042 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:15,039 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:15,097 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:16,044 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:17,042 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:18,041 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:19,043 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:19,279 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:46:19,279 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:46:19,295 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:46:19,598 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:46:19,599 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:46:20,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:20,105 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:20,599 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:21,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:22,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:22,599 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:23,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:24,044 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:24,600 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:25,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:25,354 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:26,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:27,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:27,636 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:46:27,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:27,984 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:28,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:29,045 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:30,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:30,602 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:31,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:31,285 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:32,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:33,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:34,046 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:35,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:36,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:36,993 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:37,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:38,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:39,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:40,047 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:41,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:42,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:42,209 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:42,608 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:42,984 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:42,984 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:43,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:44,048 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:45,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:46,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:47,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:47,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:48,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:49,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:50,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:51,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:52,049 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:53,050 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:53,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:46:54,050 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:55,055 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:55,628 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:46:55,628 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 with policy now +2025-01-04 20:46:55,629 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:46:55,631 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:46:55,631 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:46:55,632 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:46:55,886 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/gzbi965y-media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:46:56,050 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:56,612 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:46:56,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:56,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:46:56,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:46:56,901 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/tfqyskry-media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:46:57,056 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:57,613 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:46:57,636 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:46:57,989 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:46:57,989 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:46:58,057 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:59,056 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:46:59,076 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:00,058 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:00,614 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:01,056 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:02,054 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:03,062 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:04,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:04,615 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:05,002 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:05,065 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:06,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:07,059 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:08,059 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:09,061 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:10,060 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:11,001 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:11,060 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:12,060 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:12,987 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:12,987 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:13,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:14,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:15,068 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:16,063 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:16,347 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:47:16,347 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:47:16,348 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:16,348 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:47:16,619 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:47:16,619 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:47:16,619 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:16,862 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/skmguk37-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:47:17,066 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:18,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:18,620 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:19,067 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:20,072 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:20,620 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:21,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:22,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:22,145 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:22,621 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:23,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:24,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:25,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:26,069 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:27,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:27,637 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:47:27,638 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:27,985 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:27,986 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:28,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:29,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:30,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:31,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:32,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:33,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:33,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:34,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:35,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:36,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:37,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:38,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:39,000 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:39,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:40,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:41,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:42,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:42,986 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:42,986 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:43,072 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:44,014 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:44,078 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:44,274 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:47:44,274 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 with policy now +2025-01-04 20:47:44,281 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:47:44,282 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:47:44,282 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:47:44,283 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:47:44,558 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/31vblyuf-media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:47:44,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval +2025-01-04 20:47:44,886 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/y92u9ang-media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:47:45,084 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:45,629 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:46,085 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:46,630 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:47,084 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:47,630 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:48,106 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:48,630 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:49,088 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:50,003 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:50,086 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:51,084 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:52,087 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:52,632 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:47:53,109 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:54,099 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:55,010 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:47:55,111 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:56,111 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:57,102 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:57,637 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:47:57,994 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:47:57,994 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:47:58,107 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:47:59,105 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:00,107 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:00,143 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:01,110 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:02,111 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:03,124 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:04,115 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:05,121 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:05,420 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:05,421 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:05,421 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:05,421 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:05,447 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:48:05,447 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval +2025-01-04 20:48:05,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table +2025-01-04 20:48:05,659 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:48:05,814 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/dxaa2x5b-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:05,952 INFO wandb-upload_0:2015244 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp33fmuk32 +2025-01-04 20:48:06,081 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:48:06,091 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:48:06,112 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:06,364 INFO SenderThread:2015244 [sender.py:send_request_log_artifact():1525] logged artifact run-y2m69p6z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExNjYyNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDg1Ng==', 'latestArtifact': None}} +2025-01-04 20:48:06,364 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:48:06,365 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 20:48:06,365 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:06,365 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,365 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,365 DEBUG SenderThread:2015244 [sender.py:send():379] send: telemetry +2025-01-04 20:48:06,366 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,366 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,366 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,366 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,367 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,367 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,367 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,368 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,368 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,368 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,368 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,370 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,370 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,371 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,371 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,371 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,371 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,372 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,372 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,372 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,372 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:06,373 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:06,373 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:48:06,614 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/i2yb21h8-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:06,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:06,638 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval +2025-01-04 20:48:06,759 INFO wandb-upload_0:2015244 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdxcpnzhi +2025-01-04 20:48:06,849 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/tgrhi7ko-media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:07,113 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:07,118 INFO SenderThread:2015244 [sender.py:send_request_log_artifact():1525] logged artifact run-y2m69p6z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExNjYzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDg1OQ==', 'latestArtifact': None}} +2025-01-04 20:48:07,118 DEBUG SenderThread:2015244 [sender.py:send():379] send: files +2025-01-04 20:48:07,118 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json with policy now +2025-01-04 20:48:07,118 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:07,118 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:07,119 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:07,362 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/pl3lc1p2-media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:07,637 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:08,113 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:08,638 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:09,069 DEBUG SenderThread:2015244 [sender.py:send():379] send: exit +2025-01-04 20:48:09,069 INFO SenderThread:2015244 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:send_exit():588] handling runtime: 311 +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:send_exit():594] send defer +2025-01-04 20:48:09,070 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,070 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:48:09,070 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,070 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:48:09,071 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:48:09,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,071 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:48:09,071 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,071 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:48:09,071 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:48:09,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,071 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:48:09,071 INFO HandlerThread:2015244 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:48:09,071 DEBUG SystemMonitor:2015244 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:48:09,071 DEBUG SystemMonitor:2015244 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:48:09,072 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:48:09,072 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:48:09,123 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:48:09,123 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:48:09,124 INFO HandlerThread:2015244 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:48:09,124 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,124 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:48:09,124 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:48:09,124 DEBUG SenderThread:2015244 [sender.py:send():379] send: stats +2025-01-04 20:48:09,124 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,124 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:48:09,125 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:09,125 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:48:09,125 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:48:09,125 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:09,125 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:48:09,638 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:10,639 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:15,426 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:15,426 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:15,426 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:48:15,427 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,427 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:48:15,428 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:48:15,428 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:15,428 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:15,428 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:48:15,428 DEBUG SenderThread:2015244 [sender.py:send():379] send: history +2025-01-04 20:48:15,429 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:48:15,430 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:15,430 DEBUG SenderThread:2015244 [sender.py:send():379] send: summary +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:48:15,431 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:48:15,431 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:15,431 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:48:15,431 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,431 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:48:15,435 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:48:15,546 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:48:15,546 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:15,546 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:48:15,546 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:15,546 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:48:15,641 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:15,641 INFO Thread-12 :2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml +2025-01-04 20:48:16,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:16,762 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:48:16,762 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:16,763 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:16,763 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:48:16,763 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:16,763 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:48:16,763 INFO SenderThread:2015244 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:48:16,766 INFO SenderThread:2015244 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:48:16,768 INFO SenderThread:2015244 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:48:16,780 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:48:16,780 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:16,780 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:48:16,781 DEBUG SenderThread:2015244 [sender.py:send():379] send: artifact +2025-01-04 20:48:16,991 INFO SenderThread:2015244 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'versionIndex': 417}}} +2025-01-04 20:48:16,992 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:16,992 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:48:16,992 INFO SenderThread:2015244 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:48:17,071 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:17,642 INFO SenderThread:2015244 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:17,642 INFO SenderThread:2015244 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files +2025-01-04 20:48:17,642 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml config.yaml +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json wandb-summary.json +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt requirements.txt +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:48:17,643 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:17,646 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log output.log +2025-01-04 20:48:17,646 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/diff.patch diff.patch +2025-01-04 20:48:17,648 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 media/videos/eval/0_eval_0_4268b13f1b81246fc5ef.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 media/videos/eval/4_eval_4_b20e8a1be33290d88548.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 media/videos/eval/2_eval_2_0084bdfe26b3780f6bfb.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 media/videos/eval/3_eval_3_306a5f59512b8e5f211d.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 media/videos/eval/1_eval_1_4fd84dcdc6f80f9d2f97.mp4 +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json media/table/eval/total_success_rate_ci_6_12715ecfba84d39fa539.table.json +2025-01-04 20:48:17,651 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:48:17,651 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:17,652 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:17,652 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:48:17,652 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:17,652 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:48:17,652 INFO SenderThread:2015244 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:48:17,824 INFO wandb-upload_2:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/requirements.txt +2025-01-04 20:48:17,824 INFO wandb-upload_0:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/wandb-summary.json +2025-01-04 20:48:17,839 INFO wandb-upload_1:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/config.yaml +2025-01-04 20:48:17,890 INFO wandb-upload_4:2015244 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/files/output.log +2025-01-04 20:48:17,912 INFO wandb-upload_3:2015244 [upload_job.py:push():130] Uploaded file /tmp/tmpc1fstrumwandb/dvffvggh-events.out.tfevents.1736041377.leonmkim-ROG-Strix-G15CS-G15CS.2015050.0 +2025-01-04 20:48:18,072 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:18,072 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:18,113 INFO Thread-11 (_thread_body):2015244 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:48:18,113 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,113 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:48:18,113 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,113 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:48:18,113 INFO SenderThread:2015244 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:48:18,115 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:48:18,115 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,115 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:48:18,115 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,115 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:48:18,115 INFO SenderThread:2015244 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:48:18,312 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,312 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:48:18,312 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:48:18,312 INFO SenderThread:2015244 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:48:18,312 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send():379] send: final +2025-01-04 20:48:18,313 INFO HandlerThread:2015244 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send():379] send: footer +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: defer +2025-01-04 20:48:18,313 INFO SenderThread:2015244 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:48:18,313 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:18,313 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:18,314 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:48:18,314 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:48:18,314 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:48:18,315 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:48:18,316 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:48:18,316 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:48:18,362 DEBUG SenderThread:2015244 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:48:18,362 INFO MainThread:2015244 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:48:18,363 INFO MainThread:2015244 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:48:18,363 INFO MainThread:2015244 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:48:18,363 DEBUG HandlerThread:2015244 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:48:18,363 INFO HandlerThread:2015244 [handler.py:finish():882] shutting down handler +2025-01-04 20:48:19,314 INFO WriterThread:2015244 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/run-y2m69p6z.wandb +2025-01-04 20:48:19,362 INFO SenderThread:2015244 [sender.py:finish():1615] shutting down sender +2025-01-04 20:48:19,362 INFO SenderThread:2015244 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:48:19,363 INFO SenderThread:2015244 [file_pusher.py:join():175] waiting for file pusher diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug.log b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3ac8c622adb9ed728a458cd0a4cae6ecb524b55f --- /dev/null +++ b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Configure stats pid to 2015050 +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/settings +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug.log +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/logs/debug-internal.log +2025-01-04 20:42:57,165 INFO MainThread:2015050 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:42:57,166 INFO MainThread:2015050 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z'} +2025-01-04 20:42:57,166 INFO MainThread:2015050 [wandb_init.py:init():619] starting backend +2025-01-04 20:42:57,166 INFO MainThread:2015050 [wandb_init.py:init():623] setting up manager +2025-01-04 20:42:57,169 INFO MainThread:2015050 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:42:57,171 INFO MainThread:2015050 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:42:57,183 INFO MainThread:2015050 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:42:57,190 INFO MainThread:2015050 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:42:57,548 INFO MainThread:2015050 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:42:57,620 INFO MainThread:2015050 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:42:57,620 INFO MainThread:2015050 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:42:57,981 INFO MainThread:2015050 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:42:57,984 INFO MainThread:2015050 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:42:57,984 INFO MainThread:2015050 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/y2m69p6z/tb, True +2025-01-04 20:43:07,846 INFO MainThread:2015050 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:48:19,364 WARNING MsgRouterThr:2015050 [router.py:message_loop():77] message_loop has been closed diff --git a/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/run-y2m69p6z.wandb b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/run-y2m69p6z.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7b472458989276293a4e3b73b9816d7f742ffcc2 Binary files /dev/null and b/y2m69p6z/wandb/run-20250104_204257-y2m69p6z/run-y2m69p6z.wandb differ diff --git a/zc0axaef/.hydra/config.yaml b/zc0axaef/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/zc0axaef/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/zc0axaef/.hydra/hydra.yaml b/zc0axaef/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..337b4b534a4d4f4649acbeec41ddffe343e6cc92 --- /dev/null +++ b/zc0axaef/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/zc0axaef/.hydra/overrides.yaml b/zc0axaef/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/zc0axaef/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/zc0axaef/episode_rosbags/aligned_depth_to_color_K.npy b/zc0axaef/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/zc0axaef/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/zc0axaef/episode_rosbags/cam_tf_world.npy b/zc0axaef/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/zc0axaef/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/zc0axaef/episode_rosbags/color_K.npy b/zc0axaef/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/zc0axaef/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/zc0axaef/episode_rosbags/depth_K.npy b/zc0axaef/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/zc0axaef/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/zc0axaef/eval_robot.log b/zc0axaef/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..97177348444a11a9d38646bb13dd7ca9426f5ec5 --- /dev/null +++ b/zc0axaef/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 21:28:48,621][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:28:48,624][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/zc0axaef/eval_video/0_eval.mp4 b/zc0axaef/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..660f76eaf10ce1c6637f93aa7117dbe84e647a62 --- /dev/null +++ b/zc0axaef/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bca31b73a362a77729e614f20d459d7820c4af15f7af06473807cf40fcaafd89 +size 1458433 diff --git a/zc0axaef/eval_video/1_eval.mp4 b/zc0axaef/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7c8afa498f787d6505ce8f24f87fbd99911b68a --- /dev/null +++ b/zc0axaef/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8aad5392e5c3ede6b22c69887770e03f6219a8ecfb4b30acf1601d90181530c +size 1490887 diff --git a/zc0axaef/eval_video/2_eval.mp4 b/zc0axaef/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8436af8fd4957ebade3612d546c1cfb7b333aff --- /dev/null +++ b/zc0axaef/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1549db807633e3cb22e59d9e2084d53a8829368caba5cbd024d8a4f6ee230caf +size 1471594 diff --git a/zc0axaef/eval_video/3_eval.mp4 b/zc0axaef/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd6bffcb08ca753969d56c01e278e1a6db825f7e --- /dev/null +++ b/zc0axaef/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adddad7f6c8f1225cb18caeb495d88dc6868a23216ac7ccf5b1676cb07e94f98 +size 1480318 diff --git a/zc0axaef/eval_video/4_eval.mp4 b/zc0axaef/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9601429401f25457da4be6cb7c621e14d500637 --- /dev/null +++ b/zc0axaef/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437a72942cf548c76012c5fa67ba8f0544a503b0bf5ea4f8d6e55c29e1a1ef37 +size 1538491 diff --git a/zc0axaef/tb/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 b/zc0axaef/tb/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 new file mode 100644 index 0000000000000000000000000000000000000000..fb620041a95ee9b9e371090131a0b3f776a7ea18 --- /dev/null +++ b/zc0axaef/tb/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4cdfb2870e9c9cccee0e625695d17bf8043bad251b41e6fdf145c0ad310cbc9 +size 6539 diff --git a/zc0axaef/wandb/debug-internal.log b/zc0axaef/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..92bcb1ef3192e31a4cb0c605bf8c52e04640e868 --- /dev/null +++ b/zc0axaef/wandb/debug-internal.log @@ -0,0 +1,806 @@ +2025-01-04 21:28:54,378 INFO StreamThr :2027677 [internal.py:wandb_internal():85] W&B internal server running at pid: 2027677, started at: 2025-01-04 21:28:54.378629 +2025-01-04 21:28:54,379 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:28:54,391 INFO WriterThread:2027677 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/run-zc0axaef.wandb +2025-01-04 21:28:54,392 DEBUG SenderThread:2027677 [sender.py:send():379] send: header +2025-01-04 21:28:54,400 DEBUG SenderThread:2027677 [sender.py:send():379] send: run +2025-01-04 21:28:54,605 INFO SenderThread:2027677 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files +2025-01-04 21:28:54,605 INFO SenderThread:2027677 [sender.py:_start_run_threads():1188] run started: zc0axaef with start time 1736044134.378668 +2025-01-04 21:28:54,619 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:28:54,620 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:28:54,666 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:28:54,671 DEBUG HandlerThread:2027677 [system_info.py:__init__():26] System info init +2025-01-04 21:28:54,671 DEBUG HandlerThread:2027677 [system_info.py:__init__():41] System info init done +2025-01-04 21:28:54,671 INFO HandlerThread:2027677 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:28:54,671 INFO SystemMonitor:2027677 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:28:54,671 INFO HandlerThread:2027677 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:28:54,672 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:28:54,672 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:28:54,673 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:28:54,674 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:28:54,674 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:28:54,684 DEBUG HandlerThread:2027677 [system_info.py:probe():152] Probing system +2025-01-04 21:28:54,686 DEBUG HandlerThread:2027677 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_info.py:probe():200] Probing system done +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:28:54.684962', 'startedAt': '2025-01-05T02:28:54.369868', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3754.398875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.999, 'min': 800.0, 'max': 4700.0}, {'current': 4505.859, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 676.3531608581543}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:28:54,691 INFO HandlerThread:2027677 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:28:54,691 INFO HandlerThread:2027677 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_info.py:_save_code():44] Saving code +2025-01-04 21:28:54,692 DEBUG HandlerThread:2027677 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:28:54,693 DEBUG HandlerThread:2027677 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:28:54,754 DEBUG HandlerThread:2027677 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:28:54,754 INFO HandlerThread:2027677 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:28:54,757 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:28:54,757 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:28:54,758 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:28:54,758 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:28:55,000 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:28:55,001 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:28:55,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:28:55,001 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:28:55,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:55,004 INFO HandlerThread:2027677 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/tb" + save: true +} +control { + relay_id: "9ecfc891b2d549d5a90bbf9d6fbe4012" +} +_info { + stream_id: "zc0axaef" +} + +2025-01-04 21:28:55,004 DEBUG HandlerThread:2027677 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:28:55,027 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/qo7mtsuy-diff.patch +2025-01-04 21:28:55,033 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/umn5v2nb-wandb-metadata.json +2025-01-04 21:28:55,053 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/lhkvh1rf-code/FISH/eval_robot.py +2025-01-04 21:28:55,140 DEBUG SenderThread:2027677 [sender.py:send():379] send: telemetry +2025-01-04 21:28:55,140 DEBUG SenderThread:2027677 [sender.py:send():379] send: tbrecord +2025-01-04 21:28:55,141 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:28:55,141 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 with policy live +2025-01-04 21:28:55,337 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/faktas7x-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:28:55,606 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-metadata.json +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/diff.patch +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH/eval_robot.py +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code +2025-01-04 21:28:56,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:57,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:57,606 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:28:58,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:59,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:00,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:00,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:01,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:02,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:03,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:04,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:05,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:05,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:06,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:07,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:08,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:08,097 DEBUG SenderThread:2027677 [sender.py:send():379] send: config +2025-01-04 21:29:09,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:09,609 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:10,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:10,001 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:10,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:10,130 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:11,004 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:11,609 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:12,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:13,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:13,610 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:14,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:15,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:15,540 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:15,611 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:16,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:17,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:17,611 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:18,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:19,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:20,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:20,625 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:21,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:21,612 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:22,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:23,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:23,613 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:24,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:25,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:25,002 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:25,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:25,614 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:26,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:26,161 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:26,614 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml +2025-01-04 21:29:27,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:28,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:29,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:30,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:31,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:32,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:32,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:33,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:34,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:35,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:35,616 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:36,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:37,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:37,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:38,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:39,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:40,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:40,002 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:40,045 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:41,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:42,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:42,153 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:43,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:44,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:45,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:46,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:47,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:47,619 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:48,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:48,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:49,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:50,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:51,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:52,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:53,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:54,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:54,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:54,675 DEBUG SystemMonitor:2027677 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:29:54,675 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:29:55,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:55,002 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:55,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:56,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:57,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:58,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:59,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:59,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:00,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:00,526 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:30:00,526 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 with policy now +2025-01-04 21:30:00,527 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:30:00,530 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:30:00,530 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:30:00,530 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos +2025-01-04 21:30:00,798 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/am40xdh7-media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:30:01,015 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:01,052 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/9ijyex40-media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:30:01,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:02,016 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:03,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:04,015 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:04,140 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:05,015 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:05,625 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:06,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:07,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:08,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:09,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:09,626 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:09,894 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:10,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:10,003 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:10,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:11,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:12,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:13,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:14,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:14,628 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:30:14,865 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/xed8brqo-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:30:15,004 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:15,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:15,628 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:16,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:17,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:17,629 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:18,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:19,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:19,629 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:20,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:20,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:21,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:21,630 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:22,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:23,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:24,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:24,676 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:30:25,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:25,003 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:25,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:25,148 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:26,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:27,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:28,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:29,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:30,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:31,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:31,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:32,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:33,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:34,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:35,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:36,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:37,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:37,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:38,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:39,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:40,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:40,003 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:40,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:41,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:42,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:42,146 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:43,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:44,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:45,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:46,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:47,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:48,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:48,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:49,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:50,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:51,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:52,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:53,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:53,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:54,024 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:54,676 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:30:55,007 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:55,007 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:55,026 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:55,073 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:30:55,151 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:30:55,151 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 with policy now +2025-01-04 21:30:55,151 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:30:55,152 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:30:55,155 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:30:55,418 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/yfmv0pjf-media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:30:55,641 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:55,641 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:30:55,641 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:30:55,642 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:30:55,891 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/fk3t49rm-media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:30:56,024 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:57,025 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:57,642 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:58,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:58,155 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:59,027 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:59,642 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:00,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:01,028 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:02,028 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:03,028 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:03,541 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:03,644 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:04,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:05,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:06,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:07,039 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:08,029 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:08,440 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:31:08,440 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:31:08,441 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:31:08,645 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:31:08,645 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:31:08,876 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:09,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:09,646 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:10,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:10,014 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:10,061 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:11,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:11,646 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:12,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:13,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:13,647 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:14,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:14,588 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:15,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:16,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:17,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:18,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:19,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:20,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:20,032 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:21,032 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:22,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:23,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:23,650 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:24,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:24,677 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:31:25,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:25,004 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:25,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:25,086 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:25,651 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:26,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:27,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:28,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:29,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:30,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:31,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:31,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:32,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:33,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:34,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:35,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:36,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:37,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:37,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:38,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:39,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:40,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:40,004 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:40,045 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:41,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:42,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:42,129 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:43,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:44,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:45,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:46,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:47,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:47,203 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:48,047 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:48,296 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:31:48,296 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 with policy now +2025-01-04 21:31:48,296 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:31:48,297 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:31:48,297 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:31:48,303 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:31:48,565 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/wufjd782-media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:31:48,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:31:48,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:31:48,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:31:48,958 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/crllor0p-media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:31:49,042 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:49,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:50,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:51,043 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:52,046 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:53,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:53,047 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:53,660 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:54,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:54,677 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:31:55,004 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:55,004 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:55,046 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:56,048 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:57,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:57,661 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:58,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:58,125 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:59,048 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:00,061 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:00,277 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:32:00,277 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:32:00,278 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:32:00,662 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:32:00,663 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:32:00,904 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/ky3ddglb-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:32:01,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:01,663 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:02,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:03,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:03,353 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:03,663 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:04,054 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:05,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:05,664 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:06,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:07,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:07,665 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:08,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:09,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:09,157 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:10,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:10,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:10,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:11,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:12,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:13,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:14,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:14,961 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:15,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:15,668 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:16,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:17,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:18,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:19,052 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:20,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:20,052 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:21,052 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:22,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:23,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:24,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:24,678 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:32:25,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:25,005 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:25,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:25,115 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:26,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:27,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:28,054 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:29,054 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:30,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:31,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:31,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:32,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:33,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:34,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:35,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:36,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:36,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:37,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:38,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:39,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:40,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:40,008 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:40,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:40,850 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:32:40,850 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 with policy now +2025-01-04 21:32:40,853 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:32:40,853 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:32:40,853 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:32:40,889 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:32:41,057 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:41,123 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/e41tntkh-media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:32:41,676 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:41,677 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:32:41,677 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:32:41,677 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:32:41,903 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:41,950 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/gndxs92g-media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:32:42,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:43,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:44,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:45,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:45,678 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:46,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:47,026 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:47,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:48,073 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:49,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:49,679 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:50,080 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:51,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:51,340 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:32:51,340 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:32:51,341 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:32:51,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:51,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:32:51,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:32:52,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:52,877 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:53,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:53,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:54,085 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:54,678 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:32:55,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:55,013 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:55,075 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:55,681 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:56,084 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:57,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:57,682 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:58,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:58,430 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:59,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:59,683 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:00,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:01,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:02,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:03,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:04,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:04,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:05,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:06,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:07,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:08,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:09,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:09,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:10,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:33:10,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:33:10,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:11,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:12,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:13,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:14,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:14,145 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:15,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:16,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:17,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:18,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:19,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:20,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:20,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:21,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:22,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:23,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:24,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:24,678 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:33:25,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:33:25,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:33:25,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:25,127 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:26,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:27,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:28,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:29,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:30,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:31,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:31,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:32,080 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:33,074 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:33:33,074 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 with policy now +2025-01-04 21:33:33,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:33:33,078 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:33,079 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:33,079 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:33,088 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:33,349 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/pkioucq1-media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:33,694 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:33,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:33,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:33,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:33:33,958 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/l7gxzxhk-media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:34,090 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:35,090 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:35,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:36,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:36,090 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:37,088 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:37,696 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:38,091 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:39,096 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:40,016 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:33:40,016 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:33:40,096 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:41,094 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:41,170 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:41,698 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:42,095 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:43,097 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:44,100 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:44,406 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:44,406 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:44,406 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:44,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:44,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:44,935 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/0in4lhon-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:45,093 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:45,106 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:33:45,106 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:33:45,347 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:33:45,563 INFO wandb-upload_2:2027677 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpg1o3d5al +2025-01-04 21:33:45,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:45,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:45,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:45,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media +2025-01-04 21:33:45,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval +2025-01-04 21:33:45,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table +2025-01-04 21:33:45,763 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:33:45,771 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:33:46,005 INFO SenderThread:2027677 [sender.py:send_request_log_artifact():1525] logged artifact run-zc0axaef-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1NjEzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1Njg2Mw==', 'latestArtifact': None}} +2025-01-04 21:33:46,006 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:33:46,006 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:33:46,006 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:46,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,006 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,007 DEBUG SenderThread:2027677 [sender.py:send():379] send: telemetry +2025-01-04 21:33:46,007 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,011 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,011 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,011 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,011 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,011 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,012 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,012 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,012 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,012 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,013 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,013 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,013 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,013 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,014 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,014 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,014 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,014 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,015 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,015 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,015 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,015 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,016 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:33:46,093 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:46,192 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/a8aiq106-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:46,391 INFO wandb-upload_2:2027677 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfm19win2 +2025-01-04 21:33:46,432 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/vxolng2k-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:46,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:46,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:46,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval +2025-01-04 21:33:46,820 INFO SenderThread:2027677 [sender.py:send_request_log_artifact():1525] logged artifact run-zc0axaef-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1NjEzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1Njg2Ng==', 'latestArtifact': None}} +2025-01-04 21:33:46,823 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:46,935 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:33:46,935 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json with policy now +2025-01-04 21:33:46,935 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:46,936 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,936 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:47,004 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/ni4yenfz-media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:47,050 DEBUG SenderThread:2027677 [sender.py:send():379] send: exit +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:send_exit():588] handling runtime: 292 +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:send_exit():594] send defer +2025-01-04 21:33:47,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,050 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:33:47,051 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:33:47,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,051 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:33:47,051 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:33:47,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,051 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:33:47,051 INFO HandlerThread:2027677 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:33:47,051 DEBUG SystemMonitor:2027677 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:33:47,052 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:33:47,052 DEBUG SystemMonitor:2027677 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:33:47,052 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:33:47,106 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:33:47,106 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:33:47,106 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:33:47,107 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:33:47,107 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:33:47,107 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,107 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:33:47,107 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:33:47,107 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,107 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:33:47,249 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/jnb3r67m-media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:47,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml +2025-01-04 21:33:47,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:47,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:49,701 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:53,413 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:53,413 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:53,413 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:33:53,414 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,414 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:33:53,414 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:33:53,415 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:53,415 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,415 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:33:53,415 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:53,415 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:53,416 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:53,416 DEBUG SenderThread:2027677 [sender.py:send():379] send: summary +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:53,420 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:33:53,420 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,420 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:33:53,420 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:33:53,421 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:33:53,421 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:53,421 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,421 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:33:53,421 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,421 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:33:53,489 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:33:53,489 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,489 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:33:53,489 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,489 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:33:53,489 INFO SenderThread:2027677 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:33:53,492 INFO SenderThread:2027677 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:33:53,494 INFO SenderThread:2027677 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:33:53,503 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:33:53,503 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,503 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:33:53,503 DEBUG SenderThread:2027677 [sender.py:send():379] send: artifact +2025-01-04 21:33:53,703 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:53,703 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:53,727 INFO SenderThread:2027677 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:33:53,727 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,727 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:33:53,727 INFO SenderThread:2027677 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:33:54,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:54,703 INFO SenderThread:2027677 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml config.yaml +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json wandb-summary.json +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt requirements.txt +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:54,706 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log output.log +2025-01-04 21:33:54,706 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/diff.patch diff.patch +2025-01-04 21:33:54,706 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:33:54,714 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:54,714 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:54,714 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:33:54,715 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:54,715 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:54,715 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:33:54,715 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:54,715 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:33:54,715 INFO SenderThread:2027677 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:33:54,897 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:54,903 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml +2025-01-04 21:33:54,939 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt +2025-01-04 21:33:55,015 INFO wandb-upload_3:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/9a9sk3ue-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:55,033 INFO wandb-upload_4:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:55,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:55,051 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:55,233 INFO Thread-11 (_thread_body):2027677 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:33:55,234 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,234 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:33:55,234 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,234 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:33:55,234 INFO SenderThread:2027677 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:33:55,235 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:33:55,235 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,235 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:33:55,236 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,236 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:33:55,236 INFO SenderThread:2027677 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:33:55,451 INFO SenderThread:2027677 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:33:55,451 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:33:55,451 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,451 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:33:55,451 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,451 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:33:55,452 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:33:55,452 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,452 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:33:55,452 DEBUG SenderThread:2027677 [sender.py:send():379] send: final +2025-01-04 21:33:55,452 DEBUG SenderThread:2027677 [sender.py:send():379] send: footer +2025-01-04 21:33:55,452 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,452 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:33:55,452 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:55,453 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:55,453 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:33:55,453 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:33:55,453 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:55,455 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:33:55,455 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:33:55,456 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:55,518 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:55,519 INFO MainThread:2027677 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:33:55,519 INFO MainThread:2027677 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:33:55,520 INFO MainThread:2027677 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:33:55,520 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:33:55,520 INFO HandlerThread:2027677 [handler.py:finish():882] shutting down handler +2025-01-04 21:33:56,455 INFO WriterThread:2027677 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/run-zc0axaef.wandb +2025-01-04 21:33:56,518 INFO SenderThread:2027677 [sender.py:finish():1615] shutting down sender +2025-01-04 21:33:56,519 INFO SenderThread:2027677 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:33:56,519 INFO SenderThread:2027677 [file_pusher.py:join():175] waiting for file pusher diff --git a/zc0axaef/wandb/debug.log b/zc0axaef/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e3b6162123bcb513fbd91ee6e9add8db4e109404 --- /dev/null +++ b/zc0axaef/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Configure stats pid to 2027484 +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/settings +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug.log +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug-internal.log +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef'} +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():619] starting backend +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():623] setting up manager +2025-01-04 21:28:54,377 INFO MainThread:2027484 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:28:54,378 INFO MainThread:2027484 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:28:54,391 INFO MainThread:2027484 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:28:54,398 INFO MainThread:2027484 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:28:54,619 INFO MainThread:2027484 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:28:54,661 INFO MainThread:2027484 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:28:54,661 INFO MainThread:2027484 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:28:55,003 INFO MainThread:2027484 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:28:55,004 INFO MainThread:2027484 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/tb, True +2025-01-04 21:29:08,097 INFO MainThread:2027484 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:33:56,521 WARNING MsgRouterThr:2027484 [router.py:message_loop():77] message_loop has been closed diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH/eval_robot.py b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a4c173a4d0f7e5f65e77dbe65e11381afa27ad6 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736044134 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/diff.patch b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..31b308c90bee8b9d1f0d6472f8f51b928a966423 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[21, 45, 0.4666666666666667, 0.312052965201774, 0.6328240334721992, 0.32116228345633036, 0.6056471168783609, 0.33964740488159983, 0.5904804393468142, 0.38211534174654205, 0.5669757462904017]]} \ No newline at end of file diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..660f76eaf10ce1c6637f93aa7117dbe84e647a62 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bca31b73a362a77729e614f20d459d7820c4af15f7af06473807cf40fcaafd89 +size 1458433 diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7c8afa498f787d6505ce8f24f87fbd99911b68a --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8aad5392e5c3ede6b22c69887770e03f6219a8ecfb4b30acf1601d90181530c +size 1490887 diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8436af8fd4957ebade3612d546c1cfb7b333aff --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1549db807633e3cb22e59d9e2084d53a8829368caba5cbd024d8a4f6ee230caf +size 1471594 diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd6bffcb08ca753969d56c01e278e1a6db825f7e --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adddad7f6c8f1225cb18caeb495d88dc6868a23216ac7ccf5b1676cb07e94f98 +size 1480318 diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9601429401f25457da4be6cb7c621e14d500637 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437a72942cf548c76012c5fa67ba8f0544a503b0bf5ea4f8d6e55c29e1a1ef37 +size 1538491 diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c1fe78260c789efc0c1cbea458416c6338b5506e --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log @@ -0,0 +1,263 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736044148.097764]: resetting environment +[INFO] [1736044148.102591]: cleared current plan +[INFO] [1736044148.103489]: moving to home +[INFO] [1736044149.105470]: reached home +[INFO] [1736044149.108126]: reset action history +[INFO] [1736044151.818165]: environment reset +Starting episode 0 +[INFO] [1736044151.822616]: resetting environment +[INFO] [1736044151.827123]: cleared current plan +[INFO] [1736044151.827956]: moving to home +[INFO] [1736044152.830267]: reached home +[INFO] [1736044152.831475]: reset action history +[INFO] [1736044155.540195]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736044160.625087]: resetting environment +[INFO] [1736044160.629594]: cleared current plan +[INFO] [1736044160.631562]: moving to home +[INFO] [1736044161.635134]: reached home +[INFO] [1736044161.637245]: reset action history +[INFO] [1736044164.346193]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/episode_rosbags/episode_0_2025-01-04-21-29-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2027484.log +started roslaunch server http://158.130.50.37:40457/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2027820] +process[print_text-2]: started with pid [2027821] +process[rosbag_record-3]: started with pid [2027822] +started bagging! +For topic gripper_width: timestamp difference is 109133246 for nearest: 1736044174018240693 - target: 1736044173909107447 at idx 135 +For topic gripper_width: timestamp difference is 121058421 for nearest: 1736044184718214469 - target: 1736044184597156048 at idx 249 +[INFO] [1736044199.518344]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736044200.325322]: Stored episode 1. +[INFO] [1736044200.325593]: Saving video... +[INFO] [1736044200.515316]: Video saved! +Starting episode 1 +[INFO] [1736044200.528929]: resetting environment +[INFO] [1736044200.534697]: cleared current plan +[INFO] [1736044200.535554]: moving to home +[INFO] [1736044204.139525]: reached home +[INFO] [1736044204.139992]: reset action history +[INFO] [1736044206.851511]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044214.225857]: resetting environment +[INFO] [1736044214.234533]: cleared current plan +[INFO] [1736044214.234721]: moving to home +[INFO] [1736044215.236542]: reached home +[INFO] [1736044215.236988]: reset action history +[INFO] [1736044217.946642]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/episode_rosbags/episode_1_2025-01-04-21-30-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2027484.log +started roslaunch server http://158.130.50.37:36521/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2027982] +process[print_text-5]: started with pid [2027983] +process[rosbag_record-6]: started with pid [2027984] +started bagging! +[INFO] [1736044254.063505]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736044254.803381]: Stored episode 2. +[INFO] [1736044254.821022]: Saving video... +[INFO] [1736044255.028606]: Video saved! +Starting episode 2 +[INFO] [1736044255.069222]: resetting environment +[INFO] [1736044255.106939]: cleared current plan +[INFO] [1736044255.107306]: moving to home +[INFO] [1736044258.711441]: reached home +[INFO] [1736044258.718452]: reset action history +[INFO] [1736044261.496435]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044267.863747]: resetting environment +[INFO] [1736044267.873766]: cleared current plan +[INFO] [1736044267.874020]: moving to home +[INFO] [1736044268.875499]: reached home +[INFO] [1736044268.877127]: reset action history +[INFO] [1736044271.616207]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/episode_rosbags/episode_2_2025-01-04-21-31-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2027484.log +started roslaunch server http://158.130.50.37:35679/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2028122] +process[print_text-8]: started with pid [2028123] +process[rosbag_record-9]: started with pid [2028124] +started bagging! +For topic gripper_width: timestamp difference is 100391630 for nearest: 1736044281418237690 - target: 1736044281317846060 at idx 133 +For topic gripper_width: timestamp difference is 102854754 for nearest: 1736044284518267657 - target: 1736044284415412903 at idx 176 +[INFO] [1736044307.202492]: Storing episode... +[pub_text-7] killing on exit +[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1736044307.997958]: Stored episode 3. +[INFO] [1736044307.998844]: Saving video... +[INFO] [1736044308.231027]: Video saved! +Starting episode 3 +[INFO] [1736044308.306706]: resetting environment +[INFO] [1736044308.353530]: cleared current plan +[INFO] [1736044308.358881]: moving to home +[INFO] [1736044311.974077]: reached home +[INFO] [1736044311.977617]: reset action history +[INFO] [1736044314.913156]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044320.281832]: resetting environment +[INFO] [1736044320.302179]: cleared current plan +[INFO] [1736044320.303610]: moving to home +[INFO] [1736044321.308612]: reached home +[INFO] [1736044321.310302]: reset action history +[INFO] [1736044324.096971]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/episode_rosbags/episode_3_2025-01-04-21-32-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2027484.log +started roslaunch server http://158.130.50.37:35473/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2028278] +process[print_text-11]: started with pid [2028279] +process[rosbag_record-12]: started with pid [2028280] +started bagging! +For topic gripper_width: timestamp difference is 108554572 for nearest: 1736044332918216437 - target: 1736044332809661865 at idx 114 +[INFO] [1736044359.733633]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736044360.560284]: Stored episode 4. +[INFO] [1736044360.570458]: Saving video... +[INFO] [1736044360.786246]: Video saved! +Starting episode 4 +[INFO] [1736044360.867836]: resetting environment +[INFO] [1736044360.897949]: cleared current plan +[INFO] [1736044360.901901]: moving to home +[INFO] [1736044364.505250]: reached home +[INFO] [1736044364.511226]: reset action history +[INFO] [1736044367.330513]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044370.784561]: resetting environment +[INFO] [1736044370.817281]: cleared current plan +[INFO] [1736044370.826059]: moving to home +[INFO] [1736044371.830038]: reached home +[INFO] [1736044371.832225]: reset action history +[INFO] [1736044374.633052]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/episode_rosbags/episode_4_2025-01-04-21-32-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2027484.log +started roslaunch server http://158.130.50.37:39707/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2028431] +process[print_text-14]: started with pid [2028432] +process[rosbag_record-15]: started with pid [2028433] +started bagging! +[INFO] [1736044412.010934]: Storing episode... +[print_text-14] killing on exit[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736044412.768188]: Stored episode 5. +[INFO] [1736044412.782732]: Saving video... +[INFO] [1736044413.020527]: Video saved! +[INFO] [1736044413.081627]: resetting environment +[INFO] [1736044413.120478]: cleared current plan +[INFO] [1736044413.126635]: moving to home +[INFO] [1736044416.736671]: reached home +[INFO] [1736044416.740855]: reset action history +[INFO] [1736044419.594597]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-metadata.json b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..98675a8f966573e6f168467b4233d163a4786b0f --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:28:54.684962", + "startedAt": "2025-01-05T02:28:54.369868", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3754.398875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4501.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4505.859, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 676.3531608581543 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..80fae13a077393f871304dc6dcf0690ae8106507 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "bca31b73a362a77729e614f20d459d7820c4af15f7af06473807cf40fcaafd89", "size": 1458433, "path": "media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4"}, "global_step": 665, "_timestamp": 1736044425.6462932, "_runtime": 291.2676250934601, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "a8aad5392e5c3ede6b22c69887770e03f6219a8ecfb4b30acf1601d90181530c", "size": 1490887, "path": "media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "1549db807633e3cb22e59d9e2084d53a8829368caba5cbd024d8a4f6ee230caf", "size": 1471594, "path": "media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "adddad7f6c8f1225cb18caeb495d88dc6868a23216ac7ccf5b1676cb07e94f98", "size": 1480318, "path": "media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "437a72942cf548c76012c5fa67ba8f0544a503b0bf5ea4f8d6e55c29e1a1ef37", "size": 1538491, "path": "media/videos/eval/4_eval_4_437a72942cf548c76012.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://9mb5wru3il7d0ogx4wu978bz6ho7e2nu9ixejuutuhiju4r6urldex7xj76cbzkb5ihcz9qc2ehni9mo7ci5h7mly4vdk5u24m1pgxf8b66o31zgso30aiesisnsw5cx/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://lx3ts9yu8c60m96s2sa7nln4amwapbwo216hnphwaa5uwcikf7gpn9jos0roz3lmhhrin64ufeufcfcr9jrnxm3nik8p6z0jo3ktsxsxrt68nu8ylfxwcelldf6b9nuu:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 21, "total_num_episodes": 45, "total_success_rate": 0.4666666666666667, "total_success_rate_lb_0.01": 0.312052965201774, "total_success_rate_ub_0.01": 0.6328240334721992, "total_success_rate_lb_0.025": 0.32116228345633036, "total_success_rate_ub_0.025": 0.6056471168783609, "total_success_rate_lb_0.05": 0.33964740488159983, "total_success_rate_ub_0.05": 0.5904804393468142, "total_success_rate_lb_0.1": 0.38211534174654205, "total_success_rate_ub_0.1": 0.5669757462904017, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "fb0d58e853125c04ad7e637c3b297e1993da32b24f07ce371c813483db06c8ec", "size": 522, "artifact_path": "wandb-client-artifact://63gowk2z85lo74o1p32ufbcpbeu4zgk99utk798l3miqgr6vcrugllycafz2ljz4nr3mab65rc9cvgja3f1v9fppfau4oynqkld5ask7g79lgmc18seefrwt11gdr809/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://hio4p7k6yfhhqez7t6al7c3zx1pb6cuyeu4glbdprbcinkuxpu0utm2tvqqzrhyu6x367n9hsxyoxhbmh9sygbwn49613cydkvzkwp3uut5vrnn22av2b1z8qpruycmz:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 292}, "eval/total_success_rate": 0.46666666865348816, "eval/total_success_rate_lb_0.01": 0.31205296516418457, "eval/total_success_rate_ub_0.01": 0.6328240036964417, "eval/total_success_rate_lb_0.025": 0.32116228342056274, "eval/total_success_rate_ub_0.025": 0.605647087097168, "eval/total_success_rate_lb_0.05": 0.33964741230010986, "eval/total_success_rate_ub_0.05": 0.5904804468154907, "eval/total_success_rate_lb_0.1": 0.38211533427238464, "eval/total_success_rate_ub_0.1": 0.5669757723808289} \ No newline at end of file diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug-internal.log b/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..92bcb1ef3192e31a4cb0c605bf8c52e04640e868 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug-internal.log @@ -0,0 +1,806 @@ +2025-01-04 21:28:54,378 INFO StreamThr :2027677 [internal.py:wandb_internal():85] W&B internal server running at pid: 2027677, started at: 2025-01-04 21:28:54.378629 +2025-01-04 21:28:54,379 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:28:54,391 INFO WriterThread:2027677 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/run-zc0axaef.wandb +2025-01-04 21:28:54,392 DEBUG SenderThread:2027677 [sender.py:send():379] send: header +2025-01-04 21:28:54,400 DEBUG SenderThread:2027677 [sender.py:send():379] send: run +2025-01-04 21:28:54,605 INFO SenderThread:2027677 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files +2025-01-04 21:28:54,605 INFO SenderThread:2027677 [sender.py:_start_run_threads():1188] run started: zc0axaef with start time 1736044134.378668 +2025-01-04 21:28:54,619 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:28:54,620 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:28:54,666 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:28:54,671 DEBUG HandlerThread:2027677 [system_info.py:__init__():26] System info init +2025-01-04 21:28:54,671 DEBUG HandlerThread:2027677 [system_info.py:__init__():41] System info init done +2025-01-04 21:28:54,671 INFO HandlerThread:2027677 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:28:54,671 INFO SystemMonitor:2027677 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:28:54,671 INFO HandlerThread:2027677 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:28:54,672 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:28:54,672 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:28:54,673 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:28:54,674 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:28:54,674 INFO SystemMonitor:2027677 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:28:54,684 DEBUG HandlerThread:2027677 [system_info.py:probe():152] Probing system +2025-01-04 21:28:54,686 DEBUG HandlerThread:2027677 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_info.py:probe():200] Probing system done +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:28:54.684962', 'startedAt': '2025-01-05T02:28:54.369868', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3754.398875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.999, 'min': 800.0, 'max': 4700.0}, {'current': 4505.859, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 676.3531608581543}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:28:54,691 INFO HandlerThread:2027677 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:28:54,691 INFO HandlerThread:2027677 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:28:54,691 DEBUG HandlerThread:2027677 [system_info.py:_save_code():44] Saving code +2025-01-04 21:28:54,692 DEBUG HandlerThread:2027677 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:28:54,693 DEBUG HandlerThread:2027677 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:28:54,754 DEBUG HandlerThread:2027677 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:28:54,754 INFO HandlerThread:2027677 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:28:54,757 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:28:54,757 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:28:54,758 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:28:54,758 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:28:55,000 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:28:55,001 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:28:55,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:28:55,001 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:28:55,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:55,004 INFO HandlerThread:2027677 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/tb" + save: true +} +control { + relay_id: "9ecfc891b2d549d5a90bbf9d6fbe4012" +} +_info { + stream_id: "zc0axaef" +} + +2025-01-04 21:28:55,004 DEBUG HandlerThread:2027677 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:28:55,027 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/qo7mtsuy-diff.patch +2025-01-04 21:28:55,033 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/umn5v2nb-wandb-metadata.json +2025-01-04 21:28:55,053 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/lhkvh1rf-code/FISH/eval_robot.py +2025-01-04 21:28:55,140 DEBUG SenderThread:2027677 [sender.py:send():379] send: telemetry +2025-01-04 21:28:55,140 DEBUG SenderThread:2027677 [sender.py:send():379] send: tbrecord +2025-01-04 21:28:55,141 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:28:55,141 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 with policy live +2025-01-04 21:28:55,337 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/faktas7x-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:28:55,606 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-metadata.json +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/diff.patch +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH/eval_robot.py +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH +2025-01-04 21:28:55,607 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code +2025-01-04 21:28:56,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:57,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:57,606 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:28:58,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:59,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:00,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:00,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:01,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:02,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:03,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:04,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:05,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:05,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:06,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:07,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:08,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:08,097 DEBUG SenderThread:2027677 [sender.py:send():379] send: config +2025-01-04 21:29:09,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:09,609 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:10,001 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:10,001 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:10,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:10,130 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:11,004 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:11,609 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:12,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:13,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:13,610 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:14,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:15,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:15,540 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:15,611 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:16,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:17,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:17,611 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:18,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:19,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:20,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:20,625 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:21,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:21,612 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:22,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:23,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:23,613 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:24,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:25,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:25,002 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:25,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:25,614 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:26,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:26,161 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:26,614 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml +2025-01-04 21:29:27,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:28,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:29,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:30,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:31,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:32,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:32,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:33,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:34,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:35,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:35,616 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:36,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:37,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:37,011 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:38,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:39,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:40,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:40,002 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:40,045 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:41,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:42,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:42,153 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:43,012 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:44,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:45,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:46,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:47,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:47,619 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:29:48,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:48,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:49,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:50,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:51,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:52,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:53,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:54,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:29:54,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:54,675 DEBUG SystemMonitor:2027677 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:29:54,675 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:29:55,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:29:55,002 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:29:55,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:56,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:57,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:58,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:59,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:29:59,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:00,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:00,526 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:30:00,526 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 with policy now +2025-01-04 21:30:00,527 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:30:00,530 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:30:00,530 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:30:00,530 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:30:00,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos +2025-01-04 21:30:00,798 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/am40xdh7-media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:30:01,015 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:01,052 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/9ijyex40-media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:30:01,623 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:02,016 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:03,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:04,015 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:04,140 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:05,015 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:05,625 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:06,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:07,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:08,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:09,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:09,626 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:09,894 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:10,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:10,003 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:10,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:11,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:12,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:13,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:14,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:14,628 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:30:14,865 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/xed8brqo-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:30:15,004 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:15,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:15,628 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:16,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:17,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:17,629 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:18,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:19,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:19,629 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:20,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:20,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:21,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:21,630 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:22,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:23,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:24,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:24,676 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:30:25,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:25,003 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:25,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:25,148 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:26,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:27,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:28,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:29,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:30,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:31,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:31,019 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:32,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:33,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:34,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:35,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:36,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:37,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:37,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:38,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:39,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:40,002 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:40,003 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:40,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:41,020 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:42,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:42,146 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:43,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:44,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:45,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:46,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:47,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:48,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:48,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:49,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:50,021 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:51,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:52,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:53,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:53,022 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:54,024 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:54,676 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:30:55,007 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:30:55,007 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:30:55,026 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:55,073 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:30:55,151 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:30:55,151 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 with policy now +2025-01-04 21:30:55,151 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:30:55,152 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:30:55,155 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:30:55,418 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/yfmv0pjf-media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:30:55,641 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:55,641 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:30:55,641 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:30:55,642 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:30:55,891 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/fk3t49rm-media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:30:56,024 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:57,025 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:57,642 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:30:58,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:58,155 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:30:59,027 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:30:59,642 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:00,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:01,028 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:02,028 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:03,028 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:03,541 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:03,644 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:04,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:05,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:06,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:07,039 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:08,029 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:08,440 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:31:08,440 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:31:08,441 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:31:08,645 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:31:08,645 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:31:08,876 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:09,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:09,646 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:10,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:10,014 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:10,061 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:11,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:11,646 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:12,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:13,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:13,647 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:14,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:14,588 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:15,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:16,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:17,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:18,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:19,031 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:20,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:20,032 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:21,032 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:22,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:23,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:23,650 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:24,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:24,677 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:31:25,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:25,004 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:25,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:25,086 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:25,651 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:26,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:27,033 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:28,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:29,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:30,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:31,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:31,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:32,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:33,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:34,034 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:35,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:36,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:37,010 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:37,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:38,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:39,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:40,003 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:40,004 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:40,045 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:41,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:42,035 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:42,129 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:43,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:44,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:45,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:46,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:47,036 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:47,203 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:48,047 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:48,296 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:31:48,296 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 with policy now +2025-01-04 21:31:48,296 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:31:48,297 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:31:48,297 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:31:48,303 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:31:48,565 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/wufjd782-media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:31:48,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:31:48,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:31:48,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:31:48,958 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/crllor0p-media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:31:49,042 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:49,659 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:50,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:51,043 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:52,046 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:53,017 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:53,047 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:53,660 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:54,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:54,677 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:31:55,004 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:31:55,004 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:31:55,046 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:56,048 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:57,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:57,661 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:31:58,049 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:31:58,125 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:31:59,048 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:00,061 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:00,277 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:32:00,277 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:32:00,278 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:32:00,662 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:32:00,663 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:32:00,904 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/ky3ddglb-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:32:01,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:01,663 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:02,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:03,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:03,353 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:03,663 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:04,054 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:05,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:05,664 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:06,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:07,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:07,665 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:08,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:09,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:09,157 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:10,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:10,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:10,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:11,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:12,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:13,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:14,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:14,961 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:15,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:15,668 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:16,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:17,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:18,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:19,052 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:20,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:20,052 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:21,052 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:22,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:23,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:24,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:24,678 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:32:25,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:25,005 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:25,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:25,115 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:26,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:27,053 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:28,054 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:29,054 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:30,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:31,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:31,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:32,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:33,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:34,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:35,055 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:36,014 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:36,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:37,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:38,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:39,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:40,005 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:40,008 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:40,056 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:40,850 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:32:40,850 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 with policy now +2025-01-04 21:32:40,853 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:32:40,853 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:32:40,853 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:32:40,889 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:32:41,057 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:41,123 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/e41tntkh-media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:32:41,676 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:41,677 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:32:41,677 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:32:41,677 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:32:41,903 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:41,950 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/gndxs92g-media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:32:42,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:43,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:44,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:45,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:45,678 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:46,070 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:47,026 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:47,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:48,073 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:49,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:49,679 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:50,080 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:51,072 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:51,340 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:32:51,340 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:32:51,341 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:32:51,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:51,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:32:51,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:32:52,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:52,877 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:53,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:53,680 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:54,085 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:54,678 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:32:55,013 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:32:55,013 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:32:55,075 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:55,681 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:56,084 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:57,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:57,682 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:32:58,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:58,430 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:32:59,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:32:59,683 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:00,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:01,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:02,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:03,076 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:04,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:04,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:05,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:06,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:07,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:08,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:09,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:09,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:10,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:33:10,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:33:10,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:11,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:12,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:13,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:14,077 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:14,145 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:15,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:16,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:17,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:18,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:19,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:20,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:20,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:21,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:22,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:23,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:24,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:24,678 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:33:25,006 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:33:25,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:33:25,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:25,127 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:26,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:27,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:28,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:29,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:30,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:31,018 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:31,079 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:32,080 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:33,074 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:33:33,074 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 with policy now +2025-01-04 21:33:33,078 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:33:33,078 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:33,079 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:33,079 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:33,088 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:33,349 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/pkioucq1-media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:33,694 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:33,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:33,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:33,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval +2025-01-04 21:33:33,958 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/l7gxzxhk-media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:34,090 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:35,090 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:35,695 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:36,030 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:36,090 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:37,088 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:37,696 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:38,091 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:39,096 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:40,016 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:33:40,016 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:33:40,096 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:41,094 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:41,170 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:41,698 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:42,095 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:43,097 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:44,100 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:44,406 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:44,406 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:44,406 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:44,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:44,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:44,935 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/0in4lhon-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:45,093 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:45,106 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:33:45,106 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:33:45,347 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:33:45,563 INFO wandb-upload_2:2027677 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpg1o3d5al +2025-01-04 21:33:45,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:45,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:45,699 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:45,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media +2025-01-04 21:33:45,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval +2025-01-04 21:33:45,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table +2025-01-04 21:33:45,763 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:33:45,771 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:33:46,005 INFO SenderThread:2027677 [sender.py:send_request_log_artifact():1525] logged artifact run-zc0axaef-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1NjEzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1Njg2Mw==', 'latestArtifact': None}} +2025-01-04 21:33:46,006 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:33:46,006 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:33:46,006 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:46,006 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,006 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,007 DEBUG SenderThread:2027677 [sender.py:send():379] send: telemetry +2025-01-04 21:33:46,007 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,011 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,011 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,011 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,011 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,011 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,012 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,012 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,012 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,012 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,013 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,013 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,013 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,013 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,014 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,014 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,014 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,014 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,015 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,015 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,015 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,015 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:46,016 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:33:46,093 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:46,192 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/a8aiq106-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:46,391 INFO wandb-upload_2:2027677 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfm19win2 +2025-01-04 21:33:46,432 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/vxolng2k-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:46,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:46,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:46,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval +2025-01-04 21:33:46,820 INFO SenderThread:2027677 [sender.py:send_request_log_artifact():1525] logged artifact run-zc0axaef-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1NjEzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1Njg2Ng==', 'latestArtifact': None}} +2025-01-04 21:33:46,823 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:46,935 DEBUG SenderThread:2027677 [sender.py:send():379] send: files +2025-01-04 21:33:46,935 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json with policy now +2025-01-04 21:33:46,935 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:46,936 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:46,936 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:47,004 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/ni4yenfz-media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:47,050 DEBUG SenderThread:2027677 [sender.py:send():379] send: exit +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:send_exit():588] handling runtime: 292 +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:47,050 INFO SenderThread:2027677 [sender.py:send_exit():594] send defer +2025-01-04 21:33:47,050 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,050 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:33:47,051 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:33:47,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,051 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:33:47,051 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:33:47,051 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:33:47,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,051 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:33:47,051 INFO HandlerThread:2027677 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:33:47,051 DEBUG SystemMonitor:2027677 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:33:47,052 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:33:47,052 DEBUG SystemMonitor:2027677 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:33:47,052 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:33:47,106 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:33:47,106 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:33:47,106 INFO HandlerThread:2027677 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:33:47,107 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:33:47,107 DEBUG SenderThread:2027677 [sender.py:send():379] send: stats +2025-01-04 21:33:47,107 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,107 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:33:47,107 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:33:47,107 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:33:47,107 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:47,107 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:33:47,249 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/jnb3r67m-media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:47,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml +2025-01-04 21:33:47,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:47,700 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:49,701 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:53,413 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:53,413 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:53,413 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:33:53,414 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,414 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:33:53,414 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:33:53,415 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:53,415 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,415 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:33:53,415 DEBUG SenderThread:2027677 [sender.py:send():379] send: history +2025-01-04 21:33:53,415 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:33:53,416 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:53,416 DEBUG SenderThread:2027677 [sender.py:send():379] send: summary +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:33:53,420 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:33:53,420 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,420 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:33:53,420 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,420 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:33:53,421 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:33:53,421 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:33:53,421 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,421 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:33:53,421 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,421 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:33:53,489 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:33:53,489 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,489 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:33:53,489 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,489 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:33:53,489 INFO SenderThread:2027677 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:33:53,492 INFO SenderThread:2027677 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:33:53,494 INFO SenderThread:2027677 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:33:53,503 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:33:53,503 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:53,503 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:33:53,503 DEBUG SenderThread:2027677 [sender.py:send():379] send: artifact +2025-01-04 21:33:53,703 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:53,703 INFO Thread-12 :2027677 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:53,727 INFO SenderThread:2027677 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:33:53,727 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:53,727 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:33:53,727 INFO SenderThread:2027677 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:33:54,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:54,703 INFO SenderThread:2027677 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml config.yaml +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json wandb-summary.json +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt requirements.txt +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:33:54,704 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:54,706 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log output.log +2025-01-04 21:33:54,706 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/diff.patch diff.patch +2025-01-04 21:33:54,706 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 media/videos/eval/3_eval_3_adddad7f6c8f1225cb18.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 media/videos/eval/4_eval_4_437a72942cf548c76012.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 media/videos/eval/0_eval_0_bca31b73a362a77729e6.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 media/videos/eval/1_eval_1_a8aad5392e5c3ede6b22.mp4 +2025-01-04 21:33:54,710 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 media/videos/eval/2_eval_2_1549db807633e3cb22e5.mp4 +2025-01-04 21:33:54,714 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:33:54,714 INFO SenderThread:2027677 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json media/table/eval/total_success_rate_ci_6_fb0d58e853125c04ad7e.table.json +2025-01-04 21:33:54,714 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:33:54,715 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:54,715 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:54,715 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:33:54,715 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:54,715 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:33:54,715 INFO SenderThread:2027677 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:33:54,897 INFO wandb-upload_0:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/wandb-summary.json +2025-01-04 21:33:54,903 INFO wandb-upload_1:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/config.yaml +2025-01-04 21:33:54,939 INFO wandb-upload_2:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/requirements.txt +2025-01-04 21:33:55,015 INFO wandb-upload_3:2027677 [upload_job.py:push():130] Uploaded file /tmp/tmpp8atls2uwandb/9a9sk3ue-events.out.tfevents.1736044135.leonmkim-ROG-Strix-G15CS-G15CS.2027484.0 +2025-01-04 21:33:55,033 INFO wandb-upload_4:2027677 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/files/output.log +2025-01-04 21:33:55,051 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:55,051 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:55,233 INFO Thread-11 (_thread_body):2027677 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:33:55,234 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,234 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:33:55,234 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,234 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:33:55,234 INFO SenderThread:2027677 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:33:55,235 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:33:55,235 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,235 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:33:55,236 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,236 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:33:55,236 INFO SenderThread:2027677 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:33:55,451 INFO SenderThread:2027677 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:33:55,451 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:33:55,451 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,451 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:33:55,451 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,451 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:33:55,452 INFO SenderThread:2027677 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:33:55,452 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:33:55,452 INFO HandlerThread:2027677 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:33:55,452 DEBUG SenderThread:2027677 [sender.py:send():379] send: final +2025-01-04 21:33:55,452 DEBUG SenderThread:2027677 [sender.py:send():379] send: footer +2025-01-04 21:33:55,452 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: defer +2025-01-04 21:33:55,452 INFO SenderThread:2027677 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:33:55,452 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:55,453 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:55,453 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:33:55,453 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:33:55,453 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:33:55,455 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:33:55,455 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:33:55,456 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:33:55,518 DEBUG SenderThread:2027677 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:33:55,519 INFO MainThread:2027677 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:33:55,519 INFO MainThread:2027677 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:33:55,520 INFO MainThread:2027677 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:33:55,520 DEBUG HandlerThread:2027677 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:33:55,520 INFO HandlerThread:2027677 [handler.py:finish():882] shutting down handler +2025-01-04 21:33:56,455 INFO WriterThread:2027677 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/run-zc0axaef.wandb +2025-01-04 21:33:56,518 INFO SenderThread:2027677 [sender.py:finish():1615] shutting down sender +2025-01-04 21:33:56,519 INFO SenderThread:2027677 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:33:56,519 INFO SenderThread:2027677 [file_pusher.py:join():175] waiting for file pusher diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug.log b/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e3b6162123bcb513fbd91ee6e9add8db4e109404 --- /dev/null +++ b/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Configure stats pid to 2027484 +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/settings +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:28:54,372 INFO MainThread:2027484 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug.log +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/wandb/run-20250104_212854-zc0axaef/logs/debug-internal.log +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef'} +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():619] starting backend +2025-01-04 21:28:54,373 INFO MainThread:2027484 [wandb_init.py:init():623] setting up manager +2025-01-04 21:28:54,377 INFO MainThread:2027484 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:28:54,378 INFO MainThread:2027484 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:28:54,391 INFO MainThread:2027484 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:28:54,398 INFO MainThread:2027484 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:28:54,619 INFO MainThread:2027484 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:28:54,661 INFO MainThread:2027484 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:28:54,661 INFO MainThread:2027484 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:28:55,001 INFO MainThread:2027484 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:28:55,003 INFO MainThread:2027484 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:28:55,004 INFO MainThread:2027484 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/zc0axaef/tb, True +2025-01-04 21:29:08,097 INFO MainThread:2027484 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:33:56,521 WARNING MsgRouterThr:2027484 [router.py:message_loop():77] message_loop has been closed diff --git a/zc0axaef/wandb/run-20250104_212854-zc0axaef/run-zc0axaef.wandb b/zc0axaef/wandb/run-20250104_212854-zc0axaef/run-zc0axaef.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c26672a652e70a2e2f660fa2f1d2b2b52c64ec3d Binary files /dev/null and b/zc0axaef/wandb/run-20250104_212854-zc0axaef/run-zc0axaef.wandb differ