diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..ecd94a1f0f5d3d49901c7cb7670309f6d26c792e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,43 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcdwvdao/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +r2rih1d8/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/8m5srpvj/.hydra/config.yaml b/8m5srpvj/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/8m5srpvj/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/8m5srpvj/.hydra/hydra.yaml b/8m5srpvj/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37696d71b50c6b893ddde3b52e1c0e8afa7ce480 --- /dev/null +++ b/8m5srpvj/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/8m5srpvj/.hydra/overrides.yaml b/8m5srpvj/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/8m5srpvj/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/8m5srpvj/episode_rosbags/aligned_depth_to_color_K.npy b/8m5srpvj/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8m5srpvj/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8m5srpvj/episode_rosbags/cam_tf_world.npy b/8m5srpvj/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/8m5srpvj/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/8m5srpvj/episode_rosbags/color_K.npy b/8m5srpvj/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8m5srpvj/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8m5srpvj/episode_rosbags/depth_K.npy b/8m5srpvj/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/8m5srpvj/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/8m5srpvj/eval_robot.log b/8m5srpvj/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..028e9dfd0464e46326fc8e14be3012e4735cd9c7 --- /dev/null +++ b/8m5srpvj/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:21:07,992][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:21:07,995][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/8m5srpvj/eval_video/0_eval.mp4 b/8m5srpvj/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9b7ebe1e0288a91118e310a23c08a4741c16eb0 --- /dev/null +++ b/8m5srpvj/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:167dc5a2e4d4ef9cc88a5e7c4428d096956c4ec0ad443992e05bba23d37ca2c3 +size 1442141 diff --git a/8m5srpvj/eval_video/1_eval.mp4 b/8m5srpvj/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f1242534d0cdb6ba991ca2e3c6831269a0a65ab --- /dev/null +++ b/8m5srpvj/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9970f2300b7c91a6b22a9f0e3de0b427e6764bb8fa4c71fed7324a5beb5f1dba +size 1434541 diff --git a/8m5srpvj/eval_video/2_eval.mp4 b/8m5srpvj/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c8b88d015e35f9f8a2e422227065c2ed91281841 --- /dev/null +++ b/8m5srpvj/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21a0576511ece9efaa2544306d4bda960955c0c70b991f90bba62a94283be8e +size 1425377 diff --git a/8m5srpvj/eval_video/3_eval.mp4 b/8m5srpvj/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e7c17049698de1cbc8a83df74d1f962d49bd91d --- /dev/null +++ b/8m5srpvj/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b6e0febf1ee55734b0b08bfb4ce567f904106bdc791d15f5ca234a1ddfd1b5 +size 1444186 diff --git a/8m5srpvj/eval_video/4_eval.mp4 b/8m5srpvj/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e776df12fa9ca64db124650c479a65d74c917b57 --- /dev/null +++ b/8m5srpvj/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28b163f7f8b4ca281fa2f30c73eece9c69d477bd78cd9e200efbadba7f974efc +size 1437225 diff --git a/8m5srpvj/tb/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 b/8m5srpvj/tb/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 new file mode 100644 index 0000000000000000000000000000000000000000..d62b5cfc4ec425caeb74273750d7859ff8e0a0ec --- /dev/null +++ b/8m5srpvj/tb/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6801d960ac53a6db96dcdc9c4224050a17e4c7712f0c7659a4df2ed5902ed9 +size 6539 diff --git a/8m5srpvj/wandb/debug-internal.log b/8m5srpvj/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fd80a9dc54722ebbe27d38821c8408b5b8b32743 --- /dev/null +++ b/8m5srpvj/wandb/debug-internal.log @@ -0,0 +1,768 @@ +2025-01-06 20:21:14,045 INFO StreamThr :2108375 [internal.py:wandb_internal():85] W&B internal server running at pid: 2108375, started at: 2025-01-06 20:21:14.045359 +2025-01-06 20:21:14,046 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:21:14,068 INFO WriterThread:2108375 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/run-8m5srpvj.wandb +2025-01-06 20:21:14,068 DEBUG SenderThread:2108375 [sender.py:send():379] send: header +2025-01-06 20:21:14,076 DEBUG SenderThread:2108375 [sender.py:send():379] send: run +2025-01-06 20:21:14,323 INFO SenderThread:2108375 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files +2025-01-06 20:21:14,323 INFO SenderThread:2108375 [sender.py:_start_run_threads():1188] run started: 8m5srpvj with start time 1736212874.045614 +2025-01-06 20:21:14,338 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:21:14,338 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:21:14,411 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:21:14,416 DEBUG HandlerThread:2108375 [system_info.py:__init__():26] System info init +2025-01-06 20:21:14,416 DEBUG HandlerThread:2108375 [system_info.py:__init__():41] System info init done +2025-01-06 20:21:14,416 INFO HandlerThread:2108375 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:21:14,417 INFO SystemMonitor:2108375 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:21:14,417 INFO HandlerThread:2108375 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:21:14,418 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:21:14,419 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:21:14,419 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:21:14,420 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:21:14,420 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:21:14,431 DEBUG HandlerThread:2108375 [system_info.py:probe():152] Probing system +2025-01-06 20:21:14,432 DEBUG HandlerThread:2108375 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_info.py:probe():200] Probing system done +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:21:14.431247', 'startedAt': '2025-01-07T01:21:14.038178', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.488, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.934, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 474.64599609375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:21:14,437 INFO HandlerThread:2108375 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:21:14,437 INFO HandlerThread:2108375 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_info.py:_save_code():44] Saving code +2025-01-06 20:21:14,439 DEBUG HandlerThread:2108375 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:21:14,439 DEBUG HandlerThread:2108375 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:21:14,506 DEBUG HandlerThread:2108375 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:21:14,506 INFO HandlerThread:2108375 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:21:14,509 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:21:14,509 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:21:14,510 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:21:14,510 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:21:14,764 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:21:14,764 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:21:14,766 DEBUG SenderThread:2108375 [sender.py:send():379] send: telemetry +2025-01-06 20:21:14,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:14,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:14,767 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:14,790 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/ofuqkzqy-wandb-metadata.json +2025-01-06 20:21:14,799 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/x5187cwf-diff.patch +2025-01-06 20:21:14,804 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/j2spjoo1-code/FISH/eval_robot.py +2025-01-06 20:21:14,809 INFO HandlerThread:2108375 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/tb" + save: true +} +control { + relay_id: "b74a3821d8934e2e9d05247efac597e2" +} +_info { + stream_id: "8m5srpvj" +} + +2025-01-06 20:21:14,809 DEBUG HandlerThread:2108375 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:21:14,885 DEBUG SenderThread:2108375 [sender.py:send():379] send: tbrecord +2025-01-06 20:21:14,886 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:21:14,886 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 with policy live +2025-01-06 20:21:15,080 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/4nb2ncb5-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:21:15,325 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-metadata.json +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH/eval_robot.py +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/diff.patch +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code +2025-01-06 20:21:15,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:16,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:17,325 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:17,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:18,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:19,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:19,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:20,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:21,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:22,493 DEBUG SenderThread:2108375 [sender.py:send():379] send: config +2025-01-06 20:21:22,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:23,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:24,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:25,327 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:25,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:25,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:26,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:27,328 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:27,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:28,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:29,328 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:29,764 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:29,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:29,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:30,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:30,977 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:31,329 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:31,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:32,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:33,330 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:33,768 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:34,768 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:35,330 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:35,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:36,660 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:36,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:37,331 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:37,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:38,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:39,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:40,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:41,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:41,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:42,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:43,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:44,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:44,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:44,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:45,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:46,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:46,879 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:47,333 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml +2025-01-06 20:21:47,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:48,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:49,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:50,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:51,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:52,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:52,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:53,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:54,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:55,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:56,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:57,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:57,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:58,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:59,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:59,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:59,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:00,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:01,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:02,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:02,862 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:03,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:04,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:05,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:06,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:07,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:08,745 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:08,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:09,339 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:09,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:10,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:11,559 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:22:11,559 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 with policy now +2025-01-06 20:22:11,560 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:22:11,560 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:22:11,560 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:22:11,562 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:22:11,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:11,835 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/n0zvdo1x-media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:22:12,340 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:22:12,340 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:22:12,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos +2025-01-06 20:22:12,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:22:12,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media +2025-01-06 20:22:12,627 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/oalqn3qn-media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:22:12,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:13,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:13,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:14,420 DEBUG SystemMonitor:2108375 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:22:14,421 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:22:14,421 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:14,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:14,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:14,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:15,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:16,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:17,342 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:17,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:18,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:19,343 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:19,768 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:19,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:20,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:21,343 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:22:21,588 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/ugj6la9m-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:22:21,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:22,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:23,344 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:23,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:24,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:24,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:25,345 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:25,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:26,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:27,345 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:27,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:28,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:29,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:29,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:29,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:29,853 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:30,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:31,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:32,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:33,348 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:33,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:34,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:35,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:35,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:36,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:37,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:38,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:39,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:40,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:40,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:41,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:42,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:43,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:44,422 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:22:44,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:44,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:44,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:45,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:45,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:46,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:47,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:48,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:49,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:50,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:51,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:51,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:52,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:53,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:54,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:55,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:56,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:56,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:57,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:58,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:59,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:59,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:59,786 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:00,495 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:23:00,495 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 with policy now +2025-01-06 20:23:00,500 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:23:00,500 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:00,500 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:00,514 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:00,769 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/hj7doapu-media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:23:00,783 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:01,357 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:01,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:01,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:23:01,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:23:01,665 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/qu1v91zp-media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:23:01,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:02,517 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:02,784 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:03,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:03,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:04,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:05,359 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:05,791 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:06,792 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:07,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:08,041 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:08,792 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:09,225 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:09,225 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:09,226 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:09,360 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:09,360 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:23:09,361 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:09,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:10,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:11,361 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:11,793 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:12,793 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:13,793 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:13,991 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:14,422 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:23:14,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:14,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:14,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:15,362 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:15,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:16,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:17,363 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:17,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:18,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:19,362 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:19,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:20,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:21,364 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:21,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:22,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:23,365 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:23,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:24,557 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:24,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:25,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:26,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:27,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:28,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:29,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:29,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:29,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:29,904 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:30,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:31,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:32,796 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:33,796 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:34,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:35,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:35,799 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:36,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:37,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:38,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:39,370 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:39,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:40,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:41,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:41,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:42,802 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:43,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:44,422 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:23:44,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:44,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:44,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:45,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:46,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:46,915 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:47,802 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:48,492 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:23:48,492 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 with policy now +2025-01-06 20:23:48,495 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:23:48,495 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:48,496 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:48,507 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:48,766 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/4wtyrxzw-media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:23:48,802 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:23:49,664 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/rn6r85qf-media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:23:49,807 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:50,819 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:51,815 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:52,348 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:52,818 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:53,375 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:53,815 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:54,814 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:55,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:56,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:57,376 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:57,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:58,011 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:58,062 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:58,062 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:58,062 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:58,377 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:23:58,377 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:58,632 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/o08v9m3w-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:23:58,820 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:59,378 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:59,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:59,772 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:59,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:00,829 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:01,378 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:01,827 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:02,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:03,341 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:03,379 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:03,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:04,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:05,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:06,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:07,380 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:07,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:08,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:09,249 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:09,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:10,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:11,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:12,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:13,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:14,423 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:24:14,423 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:14,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:14,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:14,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:15,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:16,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:17,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:18,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:19,782 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:19,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:20,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:21,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:22,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:23,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:24,782 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:24,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:25,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:26,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:27,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:28,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:29,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:29,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:29,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:29,838 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:30,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:31,390 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:31,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:32,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:33,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:34,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:35,784 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:35,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:36,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:37,828 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:38,061 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:24:38,061 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:24:38,062 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 with policy now +2025-01-06 20:24:38,062 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:24:38,062 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:24:38,065 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:24:38,335 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/mpw6j1q3-media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:24:38,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:24:38,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:24:38,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:24:38,694 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/crb37em6-media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:24:38,834 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:39,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:39,833 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:40,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:40,835 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:41,833 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:42,832 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:43,394 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:43,841 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:44,423 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:24:44,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:44,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:44,830 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:45,394 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:45,839 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:45,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:46,838 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:47,124 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:24:47,124 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:24:47,125 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:24:47,395 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:47,395 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:24:47,396 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:24:47,847 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:48,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:49,396 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:49,841 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:50,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:50,986 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:51,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:52,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:53,397 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:53,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:54,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:55,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:56,786 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:56,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:57,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:58,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:59,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:59,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:59,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:00,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:01,832 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:01,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:02,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:03,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:04,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:05,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:06,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:07,787 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:07,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:08,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:09,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:10,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:11,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:12,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:12,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:13,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:14,424 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:25:14,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:25:14,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:25:14,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:15,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:16,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:17,840 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:17,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:18,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:19,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:20,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:21,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:22,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:23,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:23,846 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:24,846 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:25,846 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:26,853 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:27,156 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:25:27,156 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 with policy now +2025-01-06 20:25:27,158 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:25:27,158 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:27,158 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:27,170 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:27,409 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:27,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:27,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:27,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:25:27,434 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/796w8y4q-media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:27,706 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/isz85y39-media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:27,851 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:28,853 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:29,180 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:29,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:29,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:25:29,776 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:25:29,858 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:30,863 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:31,870 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:32,866 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:33,411 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:33,889 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:34,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:34,866 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:35,412 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:35,872 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:36,872 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:37,866 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:38,890 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:39,869 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:40,113 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:25:40,113 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:25:40,290 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval +2025-01-06 20:25:40,577 INFO wandb-upload_1:2108375 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6np0k6s9 +2025-01-06 20:25:40,609 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/ttwpb2bo-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:40,642 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:25:40,652 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:25:40,688 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/dybvvkd1-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:40,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:41,137 INFO SenderThread:2108375 [sender.py:send_request_log_artifact():1525] logged artifact run-8m5srpvj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI2MzU0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYyMTE1Mw==', 'latestArtifact': None}} +2025-01-06 20:25:41,137 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:25:41,137 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:25:41,137 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:41,137 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:41,138 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,138 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,139 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:41,139 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,143 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,143 DEBUG SenderThread:2108375 [sender.py:send():379] send: telemetry +2025-01-06 20:25:41,143 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,143 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,143 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,144 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,144 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,144 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,144 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,144 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,145 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,145 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,145 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,145 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,146 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,146 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,146 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,146 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,146 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,147 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,147 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,147 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,147 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,148 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,148 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:25:41,387 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/zof3rlsq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:41,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:41,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:41,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:41,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:41,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval +2025-01-06 20:25:41,539 INFO wandb-upload_0:2108375 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmwdv0sa_ +2025-01-06 20:25:41,640 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/vkn909sd-media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:41,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:41,915 INFO SenderThread:2108375 [sender.py:send_request_log_artifact():1525] logged artifact run-8m5srpvj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI2MzYyNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYyMTIxMA==', 'latestArtifact': None}} +2025-01-06 20:25:41,915 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:25:41,915 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json with policy now +2025-01-06 20:25:41,915 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:41,915 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,916 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:42,162 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/gzp9bf5c-media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:42,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:42,868 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:43,097 DEBUG SenderThread:2108375 [sender.py:send():379] send: exit +2025-01-06 20:25:43,097 INFO SenderThread:2108375 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:send_exit():588] handling runtime: 268 +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:send_exit():594] send defer +2025-01-06 20:25:43,098 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,098 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:25:43,098 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:25:43,099 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:25:43,099 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,099 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:25:43,099 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:25:43,099 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:25:43,100 DEBUG SystemMonitor:2108375 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:25:43,100 DEBUG SystemMonitor:2108375 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:25:43,100 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:25:43,151 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:25:43,151 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:25:43,151 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:25:43,151 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:25:43,152 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:25:43,152 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,152 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:25:43,152 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:25:43,152 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,152 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:25:43,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:43,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:45,418 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:49,201 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:49,201 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:49,201 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:25:49,202 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,202 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:25:49,202 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:25:49,202 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:49,203 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:49,203 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:25:49,203 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:49,203 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:49,204 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:49,204 DEBUG SenderThread:2108375 [sender.py:send():379] send: summary +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:49,208 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:25:49,208 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:49,208 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:25:49,208 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:25:49,212 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:49,336 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:25:49,336 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:49,337 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:25:49,337 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,337 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:25:49,420 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml +2025-01-06 20:25:49,420 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:50,099 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:51,169 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:25:51,169 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:51,169 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,169 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:25:51,170 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,170 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:25:51,170 INFO SenderThread:2108375 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:25:51,172 INFO SenderThread:2108375 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:25:51,174 INFO SenderThread:2108375 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:25:51,184 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:25:51,185 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,185 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:25:51,185 DEBUG SenderThread:2108375 [sender.py:send():379] send: artifact +2025-01-06 20:25:51,420 INFO SenderThread:2108375 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTYxMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:25:51,420 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,420 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:25:51,421 INFO SenderThread:2108375 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:25:51,422 INFO SenderThread:2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:51,422 INFO SenderThread:2108375 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files +2025-01-06 20:25:51,423 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml config.yaml +2025-01-06 20:25:51,423 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json wandb-summary.json +2025-01-06 20:25:51,423 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt requirements.txt +2025-01-06 20:25:51,425 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:25:51,425 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:51,426 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log output.log +2025-01-06 20:25:51,431 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/diff.patch diff.patch +2025-01-06 20:25:51,431 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:25:51,434 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:51,434 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:51,434 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:25:51,435 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,435 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:25:51,435 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,435 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:25:51,435 INFO SenderThread:2108375 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:25:51,619 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml +2025-01-06 20:25:51,621 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:51,645 INFO wandb-upload_4:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:51,645 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt +2025-01-06 20:25:51,737 INFO wandb-upload_3:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/00wrv8nm-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:51,938 INFO Thread-11 (_thread_body):2108375 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:25:51,938 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,938 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:25:51,938 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,938 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:25:51,938 INFO SenderThread:2108375 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:25:51,940 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:25:51,940 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,940 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:25:51,940 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,940 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:25:51,940 INFO SenderThread:2108375 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:25:52,005 INFO SenderThread:2108375 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:25:52,005 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:25:52,005 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:52,005 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:52,006 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:25:52,006 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:25:52,006 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:52,006 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send():379] send: final +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send():379] send: footer +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:52,006 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:25:52,008 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:52,008 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:25:52,009 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:25:52,010 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:52,071 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:52,071 INFO MainThread:2108375 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:25:52,072 INFO MainThread:2108375 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:25:52,072 INFO MainThread:2108375 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:25:52,072 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:25:52,072 INFO HandlerThread:2108375 [handler.py:finish():882] shutting down handler +2025-01-06 20:25:53,007 INFO WriterThread:2108375 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/run-8m5srpvj.wandb +2025-01-06 20:25:53,071 INFO SenderThread:2108375 [sender.py:finish():1615] shutting down sender +2025-01-06 20:25:53,071 INFO SenderThread:2108375 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:25:53,072 INFO SenderThread:2108375 [file_pusher.py:join():175] waiting for file pusher diff --git a/8m5srpvj/wandb/debug.log b/8m5srpvj/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..25777a116ffcaac411125550cdb1dbed6d463315 --- /dev/null +++ b/8m5srpvj/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Configure stats pid to 2108075 +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/settings +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug.log +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug-internal.log +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj'} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():619] starting backend +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():623] setting up manager +2025-01-06 20:21:14,044 INFO MainThread:2108075 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:21:14,045 INFO MainThread:2108075 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:21:14,067 INFO MainThread:2108075 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:21:14,074 INFO MainThread:2108075 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:21:14,337 INFO MainThread:2108075 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:21:14,406 INFO MainThread:2108075 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:21:14,406 INFO MainThread:2108075 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:21:14,767 INFO MainThread:2108075 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:21:14,767 INFO MainThread:2108075 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/tb, True +2025-01-06 20:21:22,492 INFO MainThread:2108075 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:25:53,073 WARNING MsgRouterThr:2108075 [router.py:message_loop():77] message_loop has been closed diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH/eval_robot.py b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db967942bdb52149d7a046cabb914e0f394ca619 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1003_0' +wandb_notes: + desc: null + value: 1003_0_req_1364_0restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736212874 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/diff.patch b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..5c450f986014be4c83153c4c0f7669b2ae0a5569 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..f19781e 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e5cb1f8a1157d60e5768aaf88f509736e552856d --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 15, 1.0, 0.7578194736918749, 1, 0.8468897341988106, 1, 0.881214111967138, 1, 0.8713090381773767, 1]]} \ No newline at end of file diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9b7ebe1e0288a91118e310a23c08a4741c16eb0 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:167dc5a2e4d4ef9cc88a5e7c4428d096956c4ec0ad443992e05bba23d37ca2c3 +size 1442141 diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f1242534d0cdb6ba991ca2e3c6831269a0a65ab --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9970f2300b7c91a6b22a9f0e3de0b427e6764bb8fa4c71fed7324a5beb5f1dba +size 1434541 diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c8b88d015e35f9f8a2e422227065c2ed91281841 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21a0576511ece9efaa2544306d4bda960955c0c70b991f90bba62a94283be8e +size 1425377 diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e7c17049698de1cbc8a83df74d1f962d49bd91d --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b6e0febf1ee55734b0b08bfb4ce567f904106bdc791d15f5ca234a1ddfd1b5 +size 1444186 diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e776df12fa9ca64db124650c479a65d74c917b57 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28b163f7f8b4ca281fa2f30c73eece9c69d477bd78cd9e200efbadba7f974efc +size 1437225 diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2e27fa9aa5322752b16718b489fa1cb4e715dd7d --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736212882.493286]: resetting environment +[INFO] [1736212882.497636]: cleared current plan +[INFO] [1736212882.498465]: moving to home +[INFO] [1736212883.500623]: reached home +[INFO] [1736212883.502030]: reset action history +[INFO] [1736212886.212822]: environment reset +Starting episode 0 +[INFO] [1736212886.216823]: resetting environment +[INFO] [1736212886.222004]: cleared current plan +[INFO] [1736212886.223237]: moving to home +[INFO] [1736212887.225258]: reached home +[INFO] [1736212887.226732]: reset action history +[INFO] [1736212889.936626]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736212891.644921]: resetting environment +[INFO] [1736212891.649400]: cleared current plan +[INFO] [1736212891.650694]: moving to home +[INFO] [1736212892.652682]: reached home +[INFO] [1736212892.654575]: reset action history +[INFO] [1736212895.365414]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/episode_rosbags/episode_0_2025-01-06-20-21-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2108075.log +started roslaunch server http://158.130.50.37:44339/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2108509] +process[print_text-2]: started with pid [2108533] +process[rosbag_record-3]: started with pid [2108534] +started bagging! +For topic gripper_width: timestamp difference is 112919946 for nearest: 1736212926699912448 - target: 1736212926586992502 at idx 249 +[INFO] [1736212930.568920]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736212931.359893]: Stored episode 1. +[INFO] [1736212931.361713]: Saving video... +[INFO] [1736212931.543830]: Video saved! +Starting episode 1 +[INFO] [1736212931.562303]: resetting environment +[INFO] [1736212931.571698]: cleared current plan +[INFO] [1736212931.572953]: moving to home +[INFO] [1736212935.276732]: reached home +[INFO] [1736212935.277155]: reset action history +[INFO] [1736212937.991747]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212940.639220]: resetting environment +[INFO] [1736212940.669495]: cleared current plan +[INFO] [1736212940.670406]: moving to home +[INFO] [1736212941.672242]: reached home +[INFO] [1736212941.672724]: reset action history +[INFO] [1736212944.389314]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/episode_rosbags/episode_1_2025-01-06-20-22-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2108075.log +started roslaunch server http://158.130.50.37:46375/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2108681] +process[print_text-5]: started with pid [2108682] +process[rosbag_record-6]: started with pid [2108683] +started bagging! +For topic gripper_width: timestamp difference is -105117380 for nearest: 1736212950833196343 - target: 1736212950938313723 at idx 90 +[INFO] [1736212979.422597]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736212980.234549]: Stored episode 2. +[INFO] [1736212980.237834]: Saving video... +[INFO] [1736212980.451371]: Video saved! +Starting episode 2 +[INFO] [1736212980.503481]: resetting environment +[INFO] [1736212980.516411]: cleared current plan +[INFO] [1736212980.516626]: moving to home +[INFO] [1736212984.221692]: reached home +[INFO] [1736212984.224495]: reset action history +[INFO] [1736212986.996218]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212988.940461]: resetting environment +[INFO] [1736212988.962104]: cleared current plan +[INFO] [1736212988.963771]: moving to home +[INFO] [1736212989.969517]: reached home +[INFO] [1736212989.974221]: reset action history +[INFO] [1736212992.717404]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/episode_rosbags/episode_2_2025-01-06-20-23-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2108075.log +started roslaunch server http://158.130.50.37:39107/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2108802] +process[print_text-8]: started with pid [2108803] +process[rosbag_record-9]: started with pid [2108827] +started bagging! +For topic gripper_width: timestamp difference is 101848719 for nearest: 1736212995533243296 - target: 1736212995431394577 at idx 38 +For topic gripper_width: timestamp difference is 102411754 for nearest: 1736212996299874551 - target: 1736212996197462797 at idx 47 +For topic gripper_width: timestamp difference is 105139857 for nearest: 1736212999699943667 - target: 1736212999594803810 at idx 87 +For topic gripper_width: timestamp difference is 106550640 for nearest: 1736213001499889797 - target: 1736213001393339157 at idx 111 +For topic gripper_width: timestamp difference is -114287499 for nearest: 1736213016633203861 - target: 1736213016747491360 at idx 249 +[INFO] [1736213027.442152]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736213028.233118]: Stored episode 3. +[INFO] [1736213028.252211]: Saving video... +[INFO] [1736213028.459419]: Video saved! +Starting episode 3 +[INFO] [1736213028.500500]: resetting environment +[INFO] [1736213028.529256]: cleared current plan +[INFO] [1736213028.533645]: moving to home +[INFO] [1736213032.348313]: reached home +[INFO] [1736213032.352512]: reset action history +[INFO] [1736213035.129218]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213038.051099]: resetting environment +[INFO] [1736213038.059865]: cleared current plan +[INFO] [1736213038.060068]: moving to home +[INFO] [1736213039.066049]: reached home +[INFO] [1736213039.067182]: reset action history +[INFO] [1736213041.849669]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/episode_rosbags/episode_3_2025-01-06-20-24-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2108075.log +started roslaunch server http://158.130.50.37:39529/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2108966] +process[print_text-11]: started with pid [2108967] +process[rosbag_record-12]: started with pid [2108968] +started bagging! +For topic gripper_width: timestamp difference is 109521445 for nearest: 1736213046199941214 - target: 1736213046090419769 at idx 56 +For topic gripper_width: timestamp difference is 161219230 for nearest: 1736213069099905124 - target: 1736213068938685894 at idx 249 +[INFO] [1736213076.906246]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736213077.757083]: Stored episode 4. +[INFO] [1736213077.763428]: Saving video... +[INFO] [1736213078.003036]: Video saved! +Starting episode 4 +[INFO] [1736213078.061170]: resetting environment +[INFO] [1736213078.085039]: cleared current plan +[INFO] [1736213078.085261]: moving to home +[INFO] [1736213081.788359]: reached home +[INFO] [1736213081.801244]: reset action history +[INFO] [1736213084.604501]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213087.085839]: resetting environment +[INFO] [1736213087.141381]: cleared current plan +[INFO] [1736213087.153263]: moving to home +[INFO] [1736213088.170182]: reached home +[INFO] [1736213088.187270]: reset action history +[INFO] [1736213090.984901]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/episode_rosbags/episode_4_2025-01-06-20-24-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2108075.log +started roslaunch server http://158.130.50.37:40675/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2109118] +process[print_text-14]: started with pid [2109119] +process[rosbag_record-15]: started with pid [2109120] +started bagging! +[INFO] [1736213126.117390]: Storing episode... +For topic gripper_width: timestamp difference is 107305015 for nearest: 1736213126133238519 - target: 1736213126025933504 at idx 249 +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736213126.886342]: Stored episode 5. +[INFO] [1736213126.891476]: Saving video... +[INFO] [1736213127.105273]: Video saved! +[INFO] [1736213127.158859]: resetting environment +[INFO] [1736213127.179950]: cleared current plan +[INFO] [1736213127.180300]: moving to home +[INFO] [1736213130.986296]: reached home +[INFO] [1736213130.990825]: reset action history +[INFO] [1736213133.840591]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-metadata.json b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f931f39d2a6492dd0b92e42ac719c0f20de43d40 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:21:14.431247", + "startedAt": "2025-01-07T01:21:14.038178", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.488, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.934, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.005, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 474.64599609375 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1f1fa428ec68ee5853fbb7dcdfdb897d6aef5f10 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "167dc5a2e4d4ef9cc88a5e7c4428d096956c4ec0ad443992e05bba23d37ca2c3", "size": 1442141, "path": "media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4"}, "global_step": 665, "_timestamp": 1736213140.568675, "_runtime": 266.5230610370636, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "9970f2300b7c91a6b22a9f0e3de0b427e6764bb8fa4c71fed7324a5beb5f1dba", "size": 1434541, "path": "media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4"}, "eval/num_success": 5.0, "episode": 15.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "a21a0576511ece9efaa2544306d4bda960955c0c70b991f90bba62a94283be8e", "size": 1425377, "path": "media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "51b6e0febf1ee55734b0b08bfb4ce567f904106bdc791d15f5ca234a1ddfd1b5", "size": 1444186, "path": "media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "28b163f7f8b4ca281fa2f30c73eece9c69d477bd78cd9e200efbadba7f974efc", "size": 1437225, "path": "media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://zdl674vlvigklvkrd54efxfdjhxhyc9vudixm0g2886ynpmxqrkinfpqc34a9bz4jt44tj7t4u41t4izmv4a9z8cgad5tz9kn14dchnej580sl16wwtlie2jhs3f8h7c/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zqf1nyuyae9uu8m3im44ed6nv59a7wqh95qc6x3tcpj1uxl1x446nb8utd43t0jfv13yun0ta7wnxutjfoadnijbfsyeqpknv4bss7ddh4gryaa8nob7bkd9cnl79y31:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 15, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.7578194736918749, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8468897341988106, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.881214111967138, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.8713090381773767, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "9da6f5d41b864e33184be83bfe8ab58f23a3014178f464aa2b357614b8c56bf4", "size": 436, "artifact_path": "wandb-client-artifact://7x7ijhbpz132krnpo9zwk963uqmaddq2c27qjdxgwviu37zox2eme5l434121vqbps3l211rmac7qlgusiwhclm8bi6dytiakan0bc8wyp37oi8uq82v0fw7vhci444b/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vyptqtvj340geoodju6ajpj4n9u1gza7b9jpsbnfcrhtb91dizk7dnrryt4nmn6j6ccpydomg2i1bxjxlrnwfe3beyapisp335iqm4f2s7syevai7cik1z1jneolkunu:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 268}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.7578194737434387, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8468897342681885, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8812140822410583, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.8713090419769287, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug-internal.log b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fd80a9dc54722ebbe27d38821c8408b5b8b32743 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug-internal.log @@ -0,0 +1,768 @@ +2025-01-06 20:21:14,045 INFO StreamThr :2108375 [internal.py:wandb_internal():85] W&B internal server running at pid: 2108375, started at: 2025-01-06 20:21:14.045359 +2025-01-06 20:21:14,046 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:21:14,068 INFO WriterThread:2108375 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/run-8m5srpvj.wandb +2025-01-06 20:21:14,068 DEBUG SenderThread:2108375 [sender.py:send():379] send: header +2025-01-06 20:21:14,076 DEBUG SenderThread:2108375 [sender.py:send():379] send: run +2025-01-06 20:21:14,323 INFO SenderThread:2108375 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files +2025-01-06 20:21:14,323 INFO SenderThread:2108375 [sender.py:_start_run_threads():1188] run started: 8m5srpvj with start time 1736212874.045614 +2025-01-06 20:21:14,338 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:21:14,338 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:21:14,411 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:21:14,416 DEBUG HandlerThread:2108375 [system_info.py:__init__():26] System info init +2025-01-06 20:21:14,416 DEBUG HandlerThread:2108375 [system_info.py:__init__():41] System info init done +2025-01-06 20:21:14,416 INFO HandlerThread:2108375 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:21:14,417 INFO SystemMonitor:2108375 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:21:14,417 INFO HandlerThread:2108375 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:21:14,418 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:21:14,419 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:21:14,419 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:21:14,420 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:21:14,420 INFO SystemMonitor:2108375 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:21:14,431 DEBUG HandlerThread:2108375 [system_info.py:probe():152] Probing system +2025-01-06 20:21:14,432 DEBUG HandlerThread:2108375 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_info.py:probe():200] Probing system done +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:21:14.431247', 'startedAt': '2025-01-07T01:21:14.038178', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.488, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.934, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 474.64599609375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:21:14,437 INFO HandlerThread:2108375 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:21:14,437 INFO HandlerThread:2108375 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:21:14,437 DEBUG HandlerThread:2108375 [system_info.py:_save_code():44] Saving code +2025-01-06 20:21:14,439 DEBUG HandlerThread:2108375 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:21:14,439 DEBUG HandlerThread:2108375 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:21:14,506 DEBUG HandlerThread:2108375 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:21:14,506 INFO HandlerThread:2108375 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:21:14,509 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:21:14,509 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:21:14,510 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:21:14,510 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:21:14,764 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:21:14,764 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:21:14,766 DEBUG SenderThread:2108375 [sender.py:send():379] send: telemetry +2025-01-06 20:21:14,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:14,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:14,767 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:14,790 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/ofuqkzqy-wandb-metadata.json +2025-01-06 20:21:14,799 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/x5187cwf-diff.patch +2025-01-06 20:21:14,804 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/j2spjoo1-code/FISH/eval_robot.py +2025-01-06 20:21:14,809 INFO HandlerThread:2108375 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/tb" + save: true +} +control { + relay_id: "b74a3821d8934e2e9d05247efac597e2" +} +_info { + stream_id: "8m5srpvj" +} + +2025-01-06 20:21:14,809 DEBUG HandlerThread:2108375 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:21:14,885 DEBUG SenderThread:2108375 [sender.py:send():379] send: tbrecord +2025-01-06 20:21:14,886 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:21:14,886 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 with policy live +2025-01-06 20:21:15,080 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/4nb2ncb5-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:21:15,325 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-metadata.json +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH/eval_robot.py +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/diff.patch +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH +2025-01-06 20:21:15,326 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code +2025-01-06 20:21:15,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:16,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:17,325 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:17,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:18,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:19,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:19,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:20,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:21,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:22,493 DEBUG SenderThread:2108375 [sender.py:send():379] send: config +2025-01-06 20:21:22,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:23,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:24,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:25,327 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:25,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:25,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:26,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:27,328 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:27,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:28,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:29,328 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:29,764 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:29,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:29,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:30,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:30,977 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:31,329 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:31,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:32,767 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:33,330 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:33,768 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:34,768 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:35,330 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:35,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:36,660 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:36,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:37,331 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:21:37,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:38,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:39,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:40,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:41,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:41,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:42,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:43,769 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:44,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:44,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:44,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:45,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:46,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:46,879 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:47,333 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml +2025-01-06 20:21:47,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:48,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:49,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:50,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:51,770 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:52,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:52,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:53,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:54,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:55,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:56,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:57,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:21:57,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:58,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:21:59,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:21:59,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:21:59,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:00,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:01,771 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:02,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:02,862 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:03,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:04,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:05,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:06,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:07,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:08,745 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:08,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:09,339 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:09,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:10,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:11,559 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:22:11,559 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 with policy now +2025-01-06 20:22:11,560 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:22:11,560 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:22:11,560 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:22:11,562 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:22:11,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:11,835 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/n0zvdo1x-media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:22:12,340 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:22:12,340 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:22:12,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos +2025-01-06 20:22:12,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:22:12,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media +2025-01-06 20:22:12,627 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/oalqn3qn-media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:22:12,774 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:13,341 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:13,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:14,420 DEBUG SystemMonitor:2108375 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:22:14,421 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:22:14,421 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:14,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:14,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:14,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:15,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:16,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:17,342 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:17,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:18,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:19,343 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:19,768 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:19,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:20,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:21,343 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:22:21,588 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/ugj6la9m-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:22:21,775 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:22,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:23,344 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:23,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:24,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:24,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:25,345 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:25,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:26,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:27,345 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:27,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:28,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:29,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:29,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:29,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:29,853 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:30,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:31,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:32,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:33,348 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:22:33,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:34,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:35,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:35,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:36,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:37,777 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:38,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:39,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:40,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:40,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:41,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:42,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:43,778 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:44,422 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:22:44,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:44,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:44,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:45,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:45,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:46,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:47,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:48,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:49,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:50,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:51,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:51,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:52,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:53,779 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:54,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:55,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:56,773 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:22:56,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:57,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:58,780 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:22:59,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:22:59,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:22:59,786 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:00,495 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:23:00,495 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 with policy now +2025-01-06 20:23:00,500 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:23:00,500 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:00,500 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:00,514 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:00,769 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/hj7doapu-media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:23:00,783 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:01,357 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:01,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:01,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:23:01,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:23:01,665 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/qu1v91zp-media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:23:01,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:02,517 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:02,784 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:03,358 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:03,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:04,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:05,359 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:05,791 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:06,792 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:07,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:08,041 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:08,792 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:09,225 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:09,225 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:09,226 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:09,360 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:09,360 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:23:09,361 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:09,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:10,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:11,361 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:11,793 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:12,793 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:13,793 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:13,991 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:14,422 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:23:14,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:14,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:14,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:15,362 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:15,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:16,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:17,363 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:17,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:18,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:19,362 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:19,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:20,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:21,364 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:21,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:22,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:23,365 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:23,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:24,557 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:24,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:25,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:26,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:27,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:28,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:29,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:29,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:29,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:29,904 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:30,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:31,795 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:32,796 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:33,796 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:34,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:35,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:35,799 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:36,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:37,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:38,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:39,370 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:39,800 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:40,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:41,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:41,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:42,802 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:43,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:44,422 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:23:44,765 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:44,765 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:44,809 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:45,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:46,801 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:46,915 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:47,802 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:48,492 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:23:48,492 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 with policy now +2025-01-06 20:23:48,495 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:23:48,495 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:48,496 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:48,507 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:48,766 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/4wtyrxzw-media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:23:48,802 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:23:49,374 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:23:49,664 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/rn6r85qf-media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:23:49,807 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:50,819 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:51,815 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:52,348 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:52,818 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:53,375 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:53,815 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:54,814 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:55,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:56,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:57,376 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:57,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:58,011 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:23:58,062 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:23:58,062 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:23:58,062 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:23:58,377 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:23:58,377 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:23:58,632 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/o08v9m3w-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:23:58,820 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:23:59,378 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:23:59,772 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:23:59,772 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:23:59,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:00,829 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:01,378 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:01,827 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:02,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:03,341 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:03,379 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:03,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:04,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:05,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:06,821 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:07,380 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:07,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:08,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:09,249 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:09,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:10,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:11,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:12,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:13,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:14,423 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:24:14,423 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:14,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:14,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:14,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:15,822 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:16,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:17,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:18,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:19,782 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:19,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:20,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:21,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:22,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:23,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:24,782 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:24,823 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:25,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:26,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:27,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:28,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:29,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:29,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:29,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:29,838 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:30,824 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:31,390 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:31,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:32,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:33,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:34,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:35,784 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:35,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:36,825 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:37,828 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:38,061 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:24:38,061 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:24:38,062 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 with policy now +2025-01-06 20:24:38,062 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:24:38,062 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:24:38,065 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:24:38,335 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/mpw6j1q3-media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:24:38,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:24:38,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:24:38,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:24:38,694 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/crb37em6-media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:24:38,834 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:39,392 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:39,833 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:40,794 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:40,835 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:41,833 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:42,832 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:43,394 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:43,841 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:44,423 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:24:44,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:44,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:44,830 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:45,394 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:45,839 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:45,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:46,838 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:47,124 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:24:47,124 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:24:47,125 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:24:47,395 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:47,395 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:24:47,396 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:24:47,847 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:48,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:49,396 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:49,841 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:50,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:50,986 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:51,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:52,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:53,397 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:24:53,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:54,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:55,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:56,786 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:24:56,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:57,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:58,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:24:59,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:24:59,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:24:59,842 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:00,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:01,832 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:01,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:02,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:03,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:04,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:05,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:06,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:07,787 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:07,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:08,843 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:09,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:10,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:11,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:12,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:12,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:13,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:14,424 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:25:14,766 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:25:14,766 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:25:14,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:15,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:16,844 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:17,840 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:17,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:18,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:19,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:20,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:21,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:22,845 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:23,788 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:23,846 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:24,846 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:25,846 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:26,853 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:27,156 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:25:27,156 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 with policy now +2025-01-06 20:25:27,158 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:25:27,158 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:27,158 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:27,170 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:27,409 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:27,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:27,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:27,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval +2025-01-06 20:25:27,434 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/796w8y4q-media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:27,706 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/isz85y39-media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:27,851 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:28,853 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:29,180 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:29,410 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:29,776 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:25:29,776 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:25:29,858 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:30,863 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:31,870 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:32,866 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:33,411 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:33,889 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:34,805 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:34,866 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:35,412 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:35,872 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:36,872 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:37,866 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:38,890 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:39,869 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:40,113 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:25:40,113 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:25:40,290 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table +2025-01-06 20:25:40,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval +2025-01-06 20:25:40,577 INFO wandb-upload_1:2108375 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6np0k6s9 +2025-01-06 20:25:40,609 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/ttwpb2bo-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:40,642 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:25:40,652 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:25:40,688 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/dybvvkd1-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:40,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:41,137 INFO SenderThread:2108375 [sender.py:send_request_log_artifact():1525] logged artifact run-8m5srpvj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI2MzU0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYyMTE1Mw==', 'latestArtifact': None}} +2025-01-06 20:25:41,137 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:25:41,137 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:25:41,137 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:41,137 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:41,138 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,138 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,139 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:41,139 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,143 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,143 DEBUG SenderThread:2108375 [sender.py:send():379] send: telemetry +2025-01-06 20:25:41,143 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,143 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,143 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,144 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,144 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,144 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,144 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,144 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,145 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,145 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,145 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,145 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,146 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,146 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,146 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,146 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,146 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,147 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,147 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,147 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,147 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,148 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:41,148 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:25:41,387 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/zof3rlsq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:41,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:41,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:41,416 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:41,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:41,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval +2025-01-06 20:25:41,539 INFO wandb-upload_0:2108375 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmwdv0sa_ +2025-01-06 20:25:41,640 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/vkn909sd-media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:41,867 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:41,915 INFO SenderThread:2108375 [sender.py:send_request_log_artifact():1525] logged artifact run-8m5srpvj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI2MzYyNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYyMTIxMA==', 'latestArtifact': None}} +2025-01-06 20:25:41,915 DEBUG SenderThread:2108375 [sender.py:send():379] send: files +2025-01-06 20:25:41,915 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json with policy now +2025-01-06 20:25:41,915 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:41,915 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:41,916 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:42,162 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/gzp9bf5c-media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:42,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:42,868 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:43,097 DEBUG SenderThread:2108375 [sender.py:send():379] send: exit +2025-01-06 20:25:43,097 INFO SenderThread:2108375 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:send_exit():588] handling runtime: 268 +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:send_exit():594] send defer +2025-01-06 20:25:43,098 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,098 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:25:43,098 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:25:43,098 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:25:43,099 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:25:43,099 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,099 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:25:43,099 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:25:43,099 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:25:43,099 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:25:43,100 DEBUG SystemMonitor:2108375 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:25:43,100 DEBUG SystemMonitor:2108375 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:25:43,100 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:25:43,151 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:25:43,151 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:25:43,151 INFO HandlerThread:2108375 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:25:43,151 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:25:43,152 DEBUG SenderThread:2108375 [sender.py:send():379] send: stats +2025-01-06 20:25:43,152 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,152 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:25:43,152 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:25:43,152 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:25:43,152 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:43,152 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:25:43,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:43,417 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:45,418 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:49,201 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:49,201 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:49,201 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:25:49,202 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,202 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:25:49,202 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:25:49,202 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:49,203 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:49,203 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:25:49,203 DEBUG SenderThread:2108375 [sender.py:send():379] send: history +2025-01-06 20:25:49,203 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:25:49,204 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:49,204 DEBUG SenderThread:2108375 [sender.py:send():379] send: summary +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:25:49,208 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:25:49,208 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:49,208 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:25:49,208 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,208 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:25:49,212 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:25:49,336 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:25:49,336 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:49,337 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:25:49,337 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:49,337 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:25:49,420 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml +2025-01-06 20:25:49,420 INFO Thread-12 :2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:50,099 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:51,169 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:25:51,169 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:51,169 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,169 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:25:51,170 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,170 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:25:51,170 INFO SenderThread:2108375 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:25:51,172 INFO SenderThread:2108375 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:25:51,174 INFO SenderThread:2108375 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:25:51,184 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:25:51,185 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,185 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:25:51,185 DEBUG SenderThread:2108375 [sender.py:send():379] send: artifact +2025-01-06 20:25:51,420 INFO SenderThread:2108375 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTYxMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:25:51,420 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,420 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:25:51,421 INFO SenderThread:2108375 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:25:51,422 INFO SenderThread:2108375 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:51,422 INFO SenderThread:2108375 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files +2025-01-06 20:25:51,423 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml config.yaml +2025-01-06 20:25:51,423 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json wandb-summary.json +2025-01-06 20:25:51,423 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt requirements.txt +2025-01-06 20:25:51,425 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:25:51,425 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:51,426 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log output.log +2025-01-06 20:25:51,431 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/diff.patch diff.patch +2025-01-06 20:25:51,431 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 media/videos/eval/4_eval_4_28b163f7f8b4ca281fa2.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 media/videos/eval/0_eval_0_167dc5a2e4d4ef9cc88a.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 media/videos/eval/3_eval_3_51b6e0febf1ee55734b0.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 media/videos/eval/1_eval_1_9970f2300b7c91a6b22a.mp4 +2025-01-06 20:25:51,433 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 media/videos/eval/2_eval_2_a21a0576511ece9efaa2.mp4 +2025-01-06 20:25:51,434 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json media/table/eval/total_success_rate_ci_6_9da6f5d41b864e33184b.table.json +2025-01-06 20:25:51,434 INFO SenderThread:2108375 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:25:51,434 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:25:51,435 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,435 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:25:51,435 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,435 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:25:51,435 INFO SenderThread:2108375 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:25:51,619 INFO wandb-upload_2:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/config.yaml +2025-01-06 20:25:51,621 INFO wandb-upload_1:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/wandb-summary.json +2025-01-06 20:25:51,645 INFO wandb-upload_4:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/output.log +2025-01-06 20:25:51,645 INFO wandb-upload_0:2108375 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/files/requirements.txt +2025-01-06 20:25:51,737 INFO wandb-upload_3:2108375 [upload_job.py:push():130] Uploaded file /tmp/tmpqznb60vhwandb/00wrv8nm-events.out.tfevents.1736212874.leonmkim-ROG-Strix-G15CS-G15CS.2108075.0 +2025-01-06 20:25:51,938 INFO Thread-11 (_thread_body):2108375 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:25:51,938 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,938 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:25:51,938 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,938 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:25:51,938 INFO SenderThread:2108375 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:25:51,940 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:25:51,940 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:51,940 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:25:51,940 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:51,940 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:25:51,940 INFO SenderThread:2108375 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:25:52,005 INFO SenderThread:2108375 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:25:52,005 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:25:52,005 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:52,005 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:52,006 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:25:52,006 INFO SenderThread:2108375 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:25:52,006 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:25:52,006 INFO HandlerThread:2108375 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send():379] send: final +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send():379] send: footer +2025-01-06 20:25:52,006 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: defer +2025-01-06 20:25:52,006 INFO SenderThread:2108375 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:25:52,007 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:25:52,008 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:52,008 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:25:52,009 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:25:52,010 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:25:52,071 DEBUG SenderThread:2108375 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:25:52,071 INFO MainThread:2108375 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:25:52,072 INFO MainThread:2108375 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:25:52,072 INFO MainThread:2108375 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:25:52,072 DEBUG HandlerThread:2108375 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:25:52,072 INFO HandlerThread:2108375 [handler.py:finish():882] shutting down handler +2025-01-06 20:25:53,007 INFO WriterThread:2108375 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/run-8m5srpvj.wandb +2025-01-06 20:25:53,071 INFO SenderThread:2108375 [sender.py:finish():1615] shutting down sender +2025-01-06 20:25:53,071 INFO SenderThread:2108375 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:25:53,072 INFO SenderThread:2108375 [file_pusher.py:join():175] waiting for file pusher diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug.log b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..25777a116ffcaac411125550cdb1dbed6d463315 --- /dev/null +++ b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Configure stats pid to 2108075 +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/settings +2025-01-06 20:21:14,039 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug.log +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/logs/debug-internal.log +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj'} +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():619] starting backend +2025-01-06 20:21:14,040 INFO MainThread:2108075 [wandb_init.py:init():623] setting up manager +2025-01-06 20:21:14,044 INFO MainThread:2108075 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:21:14,045 INFO MainThread:2108075 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:21:14,067 INFO MainThread:2108075 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:21:14,074 INFO MainThread:2108075 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:21:14,337 INFO MainThread:2108075 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:21:14,406 INFO MainThread:2108075 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:21:14,406 INFO MainThread:2108075 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:21:14,765 INFO MainThread:2108075 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:21:14,767 INFO MainThread:2108075 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:21:14,767 INFO MainThread:2108075 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/8m5srpvj/tb, True +2025-01-06 20:21:22,492 INFO MainThread:2108075 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:25:53,073 WARNING MsgRouterThr:2108075 [router.py:message_loop():77] message_loop has been closed diff --git a/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/run-8m5srpvj.wandb b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/run-8m5srpvj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..92c795ef00df88321ffb0109db9f12afb6072723 Binary files /dev/null and b/8m5srpvj/wandb/run-20250106_202114-8m5srpvj/run-8m5srpvj.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b09346e396083688edf710a93ffaf04bb750547b --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1003_0' +wandb_notes: 1003_0_req_1364_0restarted_3 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/guo9e7cv/.hydra/config.yaml b/guo9e7cv/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/guo9e7cv/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/guo9e7cv/.hydra/hydra.yaml b/guo9e7cv/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f17fa78aa46de41e8c5999c9512d711884160f8e --- /dev/null +++ b/guo9e7cv/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/guo9e7cv/.hydra/overrides.yaml b/guo9e7cv/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/guo9e7cv/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/guo9e7cv/episode_rosbags/aligned_depth_to_color_K.npy b/guo9e7cv/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/guo9e7cv/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/guo9e7cv/episode_rosbags/cam_tf_world.npy b/guo9e7cv/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/guo9e7cv/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/guo9e7cv/episode_rosbags/color_K.npy b/guo9e7cv/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/guo9e7cv/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/guo9e7cv/episode_rosbags/depth_K.npy b/guo9e7cv/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/guo9e7cv/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/guo9e7cv/eval_robot.log b/guo9e7cv/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0cd461e85fad4a665ad0b03f2b33b565b6e04d32 --- /dev/null +++ b/guo9e7cv/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 22:28:18,339][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:28:18,342][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/guo9e7cv/eval_video/0_eval.mp4 b/guo9e7cv/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a1cab1e8d65b06ae6b0009f7ddb3e0cbbac8363a --- /dev/null +++ b/guo9e7cv/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:decc75133e15da63fd7d7f4ee3ad9db1b28cf771f82c37082ca32c068f7d608f +size 1424588 diff --git a/guo9e7cv/eval_video/1_eval.mp4 b/guo9e7cv/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01582d7faa647015a576ffb8d5e781d104e8d74d --- /dev/null +++ b/guo9e7cv/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fe2f42bb4fd94a8c29e80497b5a67b0f286bc15ecf493c2e9cb437017cf993c +size 1425203 diff --git a/guo9e7cv/eval_video/2_eval.mp4 b/guo9e7cv/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8accd58ebf6bc45867967760a8b52dfa467f822 --- /dev/null +++ b/guo9e7cv/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8707cf4f2647980f40ba14f77a0918545032d8105ed728eeaeb7b469df70d5dd +size 1479829 diff --git a/guo9e7cv/eval_video/3_eval.mp4 b/guo9e7cv/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab6ba4f68989b698ab03dbca09966ddee0dd738f --- /dev/null +++ b/guo9e7cv/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ffc5d904861fc29e863c458773873c8a82715e93abebd594b53d3cee207555 +size 1486979 diff --git a/guo9e7cv/eval_video/4_eval.mp4 b/guo9e7cv/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31da592b8f43dd3200850d5890d1b280268c099d --- /dev/null +++ b/guo9e7cv/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8973ebd698ddc1cd5e389bcfb90411d7140b940551b6f9301fdd15decdd03678 +size 1523261 diff --git a/guo9e7cv/tb/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 b/guo9e7cv/tb/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 new file mode 100644 index 0000000000000000000000000000000000000000..b727006db43e38a6acf030f6fe8194db957574c4 --- /dev/null +++ b/guo9e7cv/tb/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e2be52deac17de56defc45d79d7001804f09edd41db45975a6bb4c5fd89252f +size 6539 diff --git a/guo9e7cv/wandb/debug-internal.log b/guo9e7cv/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a3827c1844adb95774b9365de6c54373c4544f96 --- /dev/null +++ b/guo9e7cv/wandb/debug-internal.log @@ -0,0 +1,868 @@ +2025-01-06 22:28:24,164 INFO StreamThr :2126082 [internal.py:wandb_internal():85] W&B internal server running at pid: 2126082, started at: 2025-01-06 22:28:24.164205 +2025-01-06 22:28:24,165 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:28:24,178 INFO WriterThread:2126082 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/run-guo9e7cv.wandb +2025-01-06 22:28:24,179 DEBUG SenderThread:2126082 [sender.py:send():379] send: header +2025-01-06 22:28:24,190 DEBUG SenderThread:2126082 [sender.py:send():379] send: run +2025-01-06 22:28:24,429 INFO SenderThread:2126082 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files +2025-01-06 22:28:24,429 INFO SenderThread:2126082 [sender.py:_start_run_threads():1188] run started: guo9e7cv with start time 1736220504.164651 +2025-01-06 22:28:24,441 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:28:24,441 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:28:24,489 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:28:24,494 DEBUG HandlerThread:2126082 [system_info.py:__init__():26] System info init +2025-01-06 22:28:24,494 DEBUG HandlerThread:2126082 [system_info.py:__init__():41] System info init done +2025-01-06 22:28:24,494 INFO HandlerThread:2126082 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:28:24,494 INFO SystemMonitor:2126082 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:28:24,495 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:28:24,495 INFO HandlerThread:2126082 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:28:24,496 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:28:24,496 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:28:24,497 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:28:24,497 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:28:24,508 DEBUG HandlerThread:2126082 [system_info.py:probe():152] Probing system +2025-01-06 22:28:24,509 DEBUG HandlerThread:2126082 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_info.py:probe():200] Probing system done +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:28:24.508306', 'startedAt': '2025-01-07T03:28:24.156158', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.232125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4496.734, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.499, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 577.9099960327148}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:28:24,514 INFO HandlerThread:2126082 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:28:24,514 INFO HandlerThread:2126082 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_info.py:_save_code():44] Saving code +2025-01-06 22:28:24,516 DEBUG HandlerThread:2126082 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:28:24,516 DEBUG HandlerThread:2126082 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:28:24,577 DEBUG HandlerThread:2126082 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:28:24,577 INFO HandlerThread:2126082 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:28:24,580 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:28:24,580 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:28:24,581 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:28:24,582 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:28:24,823 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:28:24,823 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:28:24,824 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/1ecwjq0e-diff.patch +2025-01-06 22:28:24,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:28:24,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:24,825 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:28:24,827 INFO HandlerThread:2126082 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/tb" + save: true +} +control { + relay_id: "697ef185a6b841f0aac33ae375b49a0e" +} +_info { + stream_id: "guo9e7cv" +} + +2025-01-06 22:28:24,828 DEBUG HandlerThread:2126082 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:28:24,841 INFO wandb-upload_0:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/30e3vxz6-wandb-metadata.json +2025-01-06 22:28:24,869 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/a686ncs1-code/FISH/eval_robot.py +2025-01-06 22:28:24,897 DEBUG SenderThread:2126082 [sender.py:send():379] send: telemetry +2025-01-06 22:28:24,897 DEBUG SenderThread:2126082 [sender.py:send():379] send: tbrecord +2025-01-06 22:28:24,898 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:28:24,898 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 with policy live +2025-01-06 22:28:25,080 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/tled2t0t-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-metadata.json +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/diff.patch +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH/eval_robot.py +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH +2025-01-06 22:28:25,432 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code +2025-01-06 22:28:25,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:26,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:27,430 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:27,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:28,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:29,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:29,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:30,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:31,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:32,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:33,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:34,618 DEBUG SenderThread:2126082 [sender.py:send():379] send: config +2025-01-06 22:28:34,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:34,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:35,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:36,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:37,433 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:37,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:38,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:39,434 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:39,823 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:28:39,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:28:39,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:39,917 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:40,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:41,434 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:41,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:42,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:43,435 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:43,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:44,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:45,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:45,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:46,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:47,436 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:47,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:48,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:49,437 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:49,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:50,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:51,262 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:51,438 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:51,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:52,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:53,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:54,823 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:28:54,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:28:54,865 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:55,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:56,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:56,917 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:57,439 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml +2025-01-06 22:28:57,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:58,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:59,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:00,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:01,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:02,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:02,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:03,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:04,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:05,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:06,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:07,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:07,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:08,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:09,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:09,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:09,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:10,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:11,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:12,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:12,930 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:13,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:14,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:15,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:16,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:17,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:18,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:18,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:19,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:20,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:21,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:22,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:23,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:23,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:24,497 DEBUG SystemMonitor:2126082 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:29:24,498 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:29:24,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:24,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:24,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:25,446 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:25,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:26,121 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:29:26,121 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 with policy now +2025-01-06 22:29:26,121 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:29:26,122 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:29:26,122 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:29:26,123 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:29:26,340 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/3yjde5jn-media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos +2025-01-06 22:29:26,712 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/l7hmfe2f-media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:29:26,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:27,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:27,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:28,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:29,135 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:29,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:30,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:31,448 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:31,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:32,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:33,449 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:33,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:34,489 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:34,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:35,450 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:35,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:36,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:37,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:38,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:39,451 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:29:39,690 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/ekc0u904-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:29:39,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:39,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:39,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:39,977 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:40,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:41,452 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:41,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:42,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:43,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:44,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:45,453 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:45,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:45,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:46,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:47,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:48,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:49,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:50,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:50,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:51,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:52,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:53,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:54,498 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:29:54,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:54,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:54,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:55,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:55,973 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:56,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:57,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:58,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:59,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:00,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:01,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:01,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:02,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:03,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:04,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:05,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:06,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:06,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:07,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:08,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:09,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:09,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:09,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:10,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:11,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:11,889 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:12,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:13,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:14,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:15,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:16,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:17,463 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:17,817 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:17,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:18,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:19,852 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:19,905 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:30:19,905 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:30:19,905 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 with policy now +2025-01-06 22:30:19,906 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:30:19,906 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:30:19,910 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:30:20,145 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/xf1clufe-media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:30:20,464 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:30:20,464 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:30:20,465 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:30:20,755 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/ghalz7oj-media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:30:20,856 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:21,465 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:21,852 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:22,852 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:22,921 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:23,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:24,499 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:30:24,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:24,827 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:24,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:25,466 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:25,854 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:26,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:27,466 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:27,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:28,093 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:28,853 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:29,467 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:30:29,853 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:30,250 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:30:30,250 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:30:30,251 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:30:30,468 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:30:30,856 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:31,468 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:31,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:32,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:33,854 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:34,071 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:34,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:35,469 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:35,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:36,856 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:37,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:38,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:39,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:39,827 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:39,873 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:39,925 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:40,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:41,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:42,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:43,473 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:43,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:44,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:45,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:45,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:46,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:47,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:48,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:49,475 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:49,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:50,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:50,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:51,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:52,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:53,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:54,499 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:30:54,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:54,830 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:54,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:55,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:55,875 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:56,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:57,478 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:57,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:58,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:59,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:00,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:01,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:01,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:02,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:03,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:04,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:05,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:06,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:07,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:07,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:08,872 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:09,482 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:09,730 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:31:09,730 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 with policy now +2025-01-06 22:31:09,731 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:31:09,732 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:31:09,732 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:31:09,738 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:31:09,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:09,827 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:09,872 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:09,969 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/p1u3gqbg-media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:31:10,482 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:31:10,483 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:31:10,483 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:31:10,760 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/jflcmz2z-media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:31:10,878 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:11,483 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:11,876 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:12,875 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:12,960 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:13,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:14,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:15,484 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:15,876 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:16,880 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:17,485 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:17,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:18,321 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:18,882 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:19,880 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:20,885 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:21,884 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:22,883 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:23,844 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:23,889 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:24,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:31:24,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:24,839 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:24,884 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:25,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:26,889 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:27,488 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:31:27,728 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/06xwrsc8-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:31:27,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:28,096 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:31:28,097 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:31:28,097 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:31:28,489 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:31:28,899 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:29,429 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:29,489 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:29,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:30,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:31,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:32,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:33,490 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:33,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:34,841 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:34,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:35,491 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:35,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:36,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:37,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:38,891 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:39,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:39,829 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:39,891 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:39,951 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:40,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:41,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:42,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:43,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:44,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:45,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:45,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:46,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:47,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:48,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:49,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:50,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:50,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:51,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:52,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:53,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:54,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:31:54,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:54,828 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:54,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:55,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:55,942 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:56,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:57,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:58,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:59,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:00,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:01,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:01,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:02,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:03,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:04,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:05,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:06,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:06,896 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:07,896 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:08,896 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:08,992 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:32:08,994 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:32:08,997 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 with policy now +2025-01-06 22:32:08,998 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:32:08,998 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:32:09,021 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:32:09,238 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/45ce5fpw-media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:32:09,503 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:09,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:32:09,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:32:09,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:32:09,792 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/dv1m6pkh-media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:32:09,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:09,837 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:09,899 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:10,910 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:11,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:11,901 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:11,910 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:12,913 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:13,505 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:13,913 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:14,919 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:15,505 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:15,911 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:16,915 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:17,477 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:17,506 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:17,920 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:18,929 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:19,926 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:20,924 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:21,925 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:22,853 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:22,932 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:23,934 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:24,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:32:24,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:24,833 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:24,929 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:25,928 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:26,928 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:27,914 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:27,929 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:28,928 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:29,930 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:30,932 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:31,931 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:32,936 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:33,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:33,936 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:34,937 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:35,937 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:36,939 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:37,944 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:38,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:38,945 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:39,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:39,837 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:39,943 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:40,976 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:41,961 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:42,961 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:43,962 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:43,972 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:44,968 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:45,973 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:46,964 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:47,979 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:48,974 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:49,854 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:49,978 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:50,978 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:51,975 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:52,978 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:53,981 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:54,196 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:32:54,197 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:32:54,198 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:32:54,501 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:32:54,518 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:32:54,519 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:32:54,771 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/u5kzdn73-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:32:54,840 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:54,840 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:54,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:54,995 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:55,519 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:55,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:56,981 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:57,519 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:57,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:58,993 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:59,520 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:59,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:00,575 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:00,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:01,521 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:01,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:02,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:03,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:04,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:05,849 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:05,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:06,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:07,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:08,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:09,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:09,834 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:09,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:10,984 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:10,985 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:11,984 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:12,985 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:13,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:14,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:15,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:16,851 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:16,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:17,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:18,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:19,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:20,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:21,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:22,851 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:22,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:23,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:24,501 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:33:24,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:24,834 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:24,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:25,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:26,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:27,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:28,437 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:28,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:29,530 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:29,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:30,990 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:31,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:32,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:33,851 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:33,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:34,990 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:36,002 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:36,286 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:33:36,286 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 with policy now +2025-01-06 22:33:36,292 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:33:36,292 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:36,292 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:36,292 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:36,520 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/r6avyvi4-media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:33:36,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:36,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:33:36,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:33:36,804 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/5ock1lma-media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:33:36,995 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:37,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:38,000 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:38,995 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:39,343 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:39,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:39,838 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:39,999 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:41,000 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:41,534 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:42,000 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:43,007 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:44,012 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:44,806 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:45,013 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:45,536 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:46,020 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:47,019 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:48,017 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:49,020 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:49,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:50,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:51,026 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:52,031 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:52,270 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:52,270 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:52,272 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:52,538 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:52,554 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:33:52,568 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:33:52,568 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:33:52,761 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:33:53,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:53,039 INFO wandb-upload_2:2126082 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptuu9c131 +2025-01-06 22:33:53,253 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:33:53,263 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:33:53,479 INFO SenderThread:2126082 [sender.py:send_request_log_artifact():1525] logged artifact run-guo9e7cv-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTAwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0ODU0MA==', 'latestArtifact': None}} +2025-01-06 22:33:53,479 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:33:53,479 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 22:33:53,479 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:53,479 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,500 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: telemetry +2025-01-06 22:33:53,500 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,501 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,501 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,525 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,526 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table +2025-01-06 22:33:53,540 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval +2025-01-06 22:33:53,546 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,547 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,563 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,564 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,564 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,564 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,608 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,609 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,630 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,630 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,631 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,631 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,653 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,654 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,654 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,654 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,655 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/0j382wkx-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:33:53,675 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,676 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:33:53,773 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/c4lx72ro-media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:33:53,904 INFO wandb-upload_0:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/a220e7co-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:33:54,027 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:54,050 INFO wandb-upload_1:2126082 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyjgfxxn7 +2025-01-06 22:33:54,407 INFO SenderThread:2126082 [sender.py:send_request_log_artifact():1525] logged artifact run-guo9e7cv-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTAzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0ODU0Mg==', 'latestArtifact': None}} +2025-01-06 22:33:54,407 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:33:54,407 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json with policy now +2025-01-06 22:33:54,407 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:54,408 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:54,408 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:54,502 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:33:54,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:54,676 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/wskxxezf-media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:33:54,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:54,834 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:54,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:55,030 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:55,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:56,027 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:57,028 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:58,028 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:59,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:00,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:00,630 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:00,677 DEBUG SenderThread:2126082 [sender.py:send():379] send: exit +2025-01-06 22:34:00,677 INFO SenderThread:2126082 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:send_exit():588] handling runtime: 336 +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:send_exit():594] send defer +2025-01-06 22:34:00,678 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,678 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:34:00,678 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:34:00,679 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:34:00,679 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:34:00,679 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,679 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:34:00,679 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:34:00,679 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:34:00,679 DEBUG SystemMonitor:2126082 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:34:00,679 DEBUG SystemMonitor:2126082 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:34:00,680 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:34:00,732 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:34:00,733 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:34:00,733 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:34:00,733 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,733 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:34:00,733 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:34:00,733 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:34:00,733 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,734 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:34:00,734 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,734 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:34:00,734 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:34:00,734 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,734 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:34:01,542 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:34:01,542 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:34:07,276 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:07,276 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:07,276 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:34:07,277 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,277 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:34:07,277 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:34:07,278 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:07,278 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:34:07,278 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:34:07,278 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:07,278 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:34:07,279 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:34:07,279 DEBUG SenderThread:2126082 [sender.py:send():379] send: summary +2025-01-06 22:34:07,282 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:34:07,283 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,283 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:34:07,283 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:34:07,283 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:07,283 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:34:07,283 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,283 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:34:07,287 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:07,391 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:34:07,391 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:07,391 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:34:07,392 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,392 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:34:07,544 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml +2025-01-06 22:34:07,544 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:34:07,678 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:09,241 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:34:09,241 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:09,241 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:09,241 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:34:09,242 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:09,242 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:34:09,242 INFO SenderThread:2126082 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:34:09,244 INFO SenderThread:2126082 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:34:09,246 INFO SenderThread:2126082 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:34:09,261 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:34:09,261 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:09,261 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:34:09,262 DEBUG SenderThread:2126082 [sender.py:send():379] send: artifact +2025-01-06 22:34:09,545 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:34:09,679 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:09,915 INFO wandb-upload_2:2126082 [upload_job.py:push():85] Skipped uploading /tmp/tmp_ymzf5nw/wandb-job.json +2025-01-06 22:34:09,916 INFO wandb-upload_0:2126082 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4m9y5tu9 +2025-01-06 22:34:10,037 INFO wandb-upload_1:2126082 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpsg96ts72 +2025-01-06 22:34:10,815 INFO SenderThread:2126082 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:34:10,815 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:10,816 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:34:10,816 INFO SenderThread:2126082 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:34:11,546 INFO SenderThread:2126082 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml config.yaml +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json wandb-summary.json +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt requirements.txt +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log output.log +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/diff.patch diff.patch +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:34:11,554 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:34:11,554 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:11,555 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:11,555 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:34:11,556 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:11,556 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:34:11,556 INFO SenderThread:2126082 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:34:11,680 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:11,680 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:11,718 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml +2025-01-06 22:34:11,739 INFO wandb-upload_0:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:34:11,744 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt +2025-01-06 22:34:11,778 INFO wandb-upload_4:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:34:11,832 INFO wandb-upload_3:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/vp03xi33-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:34:12,032 INFO Thread-11 (_thread_body):2126082 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:34:12,032 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,032 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:34:12,032 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,033 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:34:12,033 INFO SenderThread:2126082 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:34:12,034 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:34:12,034 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,034 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:34:12,035 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,035 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:34:12,035 INFO SenderThread:2126082 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:34:12,099 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,099 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:34:12,099 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:34:12,099 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,100 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send():379] send: final +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send():379] send: footer +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,100 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:34:12,100 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:12,101 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:34:12,101 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:34:12,102 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:12,103 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:34:12,103 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:34:12,104 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:12,148 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:12,149 INFO MainThread:2126082 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:34:12,149 INFO MainThread:2126082 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:34:12,149 INFO MainThread:2126082 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:34:12,150 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:34:12,150 INFO HandlerThread:2126082 [handler.py:finish():882] shutting down handler +2025-01-06 22:34:13,104 INFO WriterThread:2126082 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/run-guo9e7cv.wandb +2025-01-06 22:34:13,149 INFO SenderThread:2126082 [sender.py:finish():1615] shutting down sender +2025-01-06 22:34:13,149 INFO SenderThread:2126082 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:34:13,149 INFO SenderThread:2126082 [file_pusher.py:join():175] waiting for file pusher diff --git a/guo9e7cv/wandb/debug.log b/guo9e7cv/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0c40cadc2dee5989a88495ff31e229133cd993c4 --- /dev/null +++ b/guo9e7cv/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Configure stats pid to 2125886 +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/settings +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug.log +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug-internal.log +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:28:24,159 INFO MainThread:2125886 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv'} +2025-01-06 22:28:24,159 INFO MainThread:2125886 [wandb_init.py:init():619] starting backend +2025-01-06 22:28:24,159 INFO MainThread:2125886 [wandb_init.py:init():623] setting up manager +2025-01-06 22:28:24,163 INFO MainThread:2125886 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:28:24,164 INFO MainThread:2125886 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:28:24,177 INFO MainThread:2125886 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:28:24,185 INFO MainThread:2125886 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:28:24,440 INFO MainThread:2125886 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:28:24,479 INFO MainThread:2125886 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:28:24,479 INFO MainThread:2125886 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:28:24,823 INFO MainThread:2125886 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:28:24,824 INFO MainThread:2125886 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:28:24,824 INFO MainThread:2125886 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:28:24,824 INFO MainThread:2125886 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:28:24,825 INFO MainThread:2125886 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:28:24,825 INFO MainThread:2125886 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/tb, True +2025-01-06 22:28:34,618 INFO MainThread:2125886 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:34:13,150 WARNING MsgRouterThr:2125886 [router.py:message_loop():77] message_loop has been closed diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH/eval_robot.py b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f061051e074fee8dc866c4193d0e9d520ef2f2b --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1003_0' +wandb_notes: + desc: null + value: 1003_0_req_1364_0restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736220504 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/diff.patch b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c03766eedde4e00adf1f118f93a1d1dd2d31994e --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..1643b06 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d97d19278524385d0560402d6e8edb83b8ff67c9 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 25, 1.0, 0.8467192947171094, 1, 0.9050990789318509, 1, 0.9269341155032782, 1, 0.9206686019056123, 1]]} \ No newline at end of file diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a1cab1e8d65b06ae6b0009f7ddb3e0cbbac8363a --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:decc75133e15da63fd7d7f4ee3ad9db1b28cf771f82c37082ca32c068f7d608f +size 1424588 diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01582d7faa647015a576ffb8d5e781d104e8d74d --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fe2f42bb4fd94a8c29e80497b5a67b0f286bc15ecf493c2e9cb437017cf993c +size 1425203 diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8accd58ebf6bc45867967760a8b52dfa467f822 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8707cf4f2647980f40ba14f77a0918545032d8105ed728eeaeb7b469df70d5dd +size 1479829 diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab6ba4f68989b698ab03dbca09966ddee0dd738f --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ffc5d904861fc29e863c458773873c8a82715e93abebd594b53d3cee207555 +size 1486979 diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31da592b8f43dd3200850d5890d1b280268c099d --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8973ebd698ddc1cd5e389bcfb90411d7140b940551b6f9301fdd15decdd03678 +size 1523261 diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5ef0edabcfe70b9ccc765f37d7ffc9b25f180777 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log @@ -0,0 +1,265 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736220514.618958]: resetting environment +[INFO] [1736220514.623875]: cleared current plan +[INFO] [1736220514.624766]: moving to home +[INFO] [1736220515.627574]: reached home +[INFO] [1736220515.629017]: reset action history +[INFO] [1736220518.389169]: environment reset +Starting episode 0 +[INFO] [1736220518.393468]: resetting environment +[INFO] [1736220518.397924]: cleared current plan +[INFO] [1736220518.399156]: moving to home +[INFO] [1736220519.401372]: reached home +[INFO] [1736220519.402659]: reset action history +[INFO] [1736220522.113661]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736220526.234331]: resetting environment +[INFO] [1736220526.243059]: cleared current plan +[INFO] [1736220526.244582]: moving to home +[INFO] [1736220527.247453]: reached home +[INFO] [1736220527.249233]: reset action history +[INFO] [1736220529.958699]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/episode_rosbags/episode_0_2025-01-06-22-28-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2125886.log +started roslaunch server http://158.130.50.37:46591/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2126196] +process[print_text-2]: started with pid [2126220] +process[rosbag_record-3]: started with pid [2126221] +started bagging! +For topic gripper_width: timestamp difference is 104387693 for nearest: 1736220564157393388 - target: 1736220564053005695 at idx 249 +[INFO] [1736220565.183645]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736220565.900346]: Stored episode 1. +[INFO] [1736220565.900978]: Saving video... +[INFO] [1736220566.104975]: Video saved! +Starting episode 1 +[INFO] [1736220566.125625]: resetting environment +[INFO] [1736220566.134132]: cleared current plan +[INFO] [1736220566.135002]: moving to home +[INFO] [1736220569.737301]: reached home +[INFO] [1736220569.738015]: reset action history +[INFO] [1736220572.448294]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220579.285124]: resetting environment +[INFO] [1736220579.289719]: cleared current plan +[INFO] [1736220579.289956]: moving to home +[INFO] [1736220580.292490]: reached home +[INFO] [1736220580.292942]: reset action history +[INFO] [1736220583.002897]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/episode_rosbags/episode_1_2025-01-06-22-29-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2125886.log +started roslaunch server http://158.130.50.37:46607/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2126369] +process[print_text-5]: started with pid [2126370] +process[rosbag_record-6]: started with pid [2126371] +started bagging! +For topic gripper_width: timestamp difference is 112765181 for nearest: 1736220615757423747 - target: 1736220615644658566 at idx 249 +[INFO] [1736220618.913495]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736220619.658525]: Stored episode 2. +[INFO] [1736220619.662592]: Saving video... +[INFO] [1736220619.862858]: Video saved! +Starting episode 2 +[INFO] [1736220619.893914]: resetting environment +[INFO] [1736220619.915436]: cleared current plan +[INFO] [1736220619.919046]: moving to home +[INFO] [1736220623.325248]: reached home +[INFO] [1736220623.328004]: reset action history +[INFO] [1736220626.049216]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220629.064789]: resetting environment +[INFO] [1736220629.076734]: cleared current plan +[INFO] [1736220629.076992]: moving to home +[INFO] [1736220630.078491]: reached home +[INFO] [1736220630.082786]: reset action history +[INFO] [1736220632.813983]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/episode_rosbags/episode_2_2025-01-06-22-30-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2125886.log +started roslaunch server http://158.130.50.37:42421/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2126515] +process[print_text-8]: started with pid [2126516] +process[rosbag_record-9]: started with pid [2126517] +started bagging! +For topic gripper_width: timestamp difference is -100351769 for nearest: 1736220640590726179 - target: 1736220640691077948 at idx 110 +For topic gripper_width: timestamp difference is 105275051 for nearest: 1736220647790722267 - target: 1736220647685447216 at idx 204 +For topic gripper_width: timestamp difference is 111689698 for nearest: 1736220655590724122 - target: 1736220655479034424 at idx 249 +For topic gripper_width: timestamp difference is 111865761 for nearest: 1736220655857352259 - target: 1736220655745486498 at idx 249 +For topic gripper_width: timestamp difference is 106933809 for nearest: 1736220656357395626 - target: 1736220656250461817 at idx 249 +[INFO] [1736220668.561077]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736220669.413464]: Stored episode 3. +[INFO] [1736220669.416941]: Saving video... +[INFO] [1736220669.688390]: Video saved! +Starting episode 3 +[INFO] [1736220669.742281]: resetting environment +[INFO] [1736220669.774337]: cleared current plan +[INFO] [1736220669.774929]: moving to home +[INFO] [1736220673.378410]: reached home +[INFO] [1736220673.382553]: reset action history +[INFO] [1736220676.273118]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220687.360465]: resetting environment +[INFO] [1736220687.380449]: cleared current plan +[INFO] [1736220687.382128]: moving to home +[INFO] [1736220688.388247]: reached home +[INFO] [1736220688.393217]: reset action history +[INFO] [1736220691.127127]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/episode_rosbags/episode_3_2025-01-06-22-31-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2125886.log +started roslaunch server http://158.130.50.37:33759/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2126656] +process[print_text-11]: started with pid [2126657] +process[rosbag_record-12]: started with pid [2126658] +started bagging! +[INFO] [1736220727.943209]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736220728.718667]: Stored episode 4. +[INFO] [1736220728.729688]: Saving video... +[INFO] [1736220728.946563]: Video saved! +Starting episode 4 +[INFO] [1736220728.998437]: resetting environment +[INFO] [1736220729.026449]: cleared current plan +[INFO] [1736220729.026798]: moving to home +[INFO] [1736220732.629119]: reached home +[INFO] [1736220732.635044]: reset action history +[INFO] [1736220735.429567]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220774.101280]: resetting environment +[INFO] [1736220774.134519]: cleared current plan +[INFO] [1736220774.142778]: moving to home +[INFO] [1736220775.146130]: reached home +[INFO] [1736220775.148325]: reset action history +[INFO] [1736220777.942858]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/episode_rosbags/episode_4_2025-01-06-22-32-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2125886.log +started roslaunch server http://158.130.50.37:36443/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2126887] +process[print_text-14]: started with pid [2126888] +process[rosbag_record-15]: started with pid [2126889] +started bagging! +For topic gripper_width: timestamp difference is 103561074 for nearest: 1736220807390681655 - target: 1736220807287120581 at idx 249 +[INFO] [1736220815.189068]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736220815.996155]: Stored episode 5. +[INFO] [1736220815.999928]: Saving video... +[INFO] [1736220816.235525]: Video saved! +[INFO] [1736220816.292439]: resetting environment +[INFO] [1736220816.339447]: cleared current plan +[INFO] [1736220816.341290]: moving to home +[INFO] [1736220819.949984]: reached home +[INFO] [1736220819.955379]: reset action history +[INFO] [1736220822.754792]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-metadata.json b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..06359d9b056ce228ba68e3c5df0c5c61a4532f0b --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T03:28:24.508306", + "startedAt": "2025-01-07T03:28:24.156158", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.232125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4496.734, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.499, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 577.9099960327148 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0a5d98f4f6f3616a3a7b900d651b2c3bed5435ed --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "decc75133e15da63fd7d7f4ee3ad9db1b28cf771f82c37082ca32c068f7d608f", "size": 1424588, "path": "media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4"}, "global_step": 665, "_timestamp": 1736220833.050027, "_runtime": 328.8853759765625, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "7fe2f42bb4fd94a8c29e80497b5a67b0f286bc15ecf493c2e9cb437017cf993c", "size": 1425203, "path": "media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4"}, "eval/num_success": 5.0, "episode": 25.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "8707cf4f2647980f40ba14f77a0918545032d8105ed728eeaeb7b469df70d5dd", "size": 1479829, "path": "media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "81ffc5d904861fc29e863c458773873c8a82715e93abebd594b53d3cee207555", "size": 1486979, "path": "media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "8973ebd698ddc1cd5e389bcfb90411d7140b940551b6f9301fdd15decdd03678", "size": 1523261, "path": "media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://2k4kyd44xmtl29eijqbo2vlw4n9w98lwdwdqkvejfhmuecry1jdrnf9lg3pu1n06v646euuuj1i5611w6k4kmgqzkhhyftj9ozrysovr7zl7zm76o5mau7wi3ad7t24o/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://frcjyf5kxja4e02xpxw9ujh3oj0cnn3aan275cs94kticcdn1pb2afe0ji97od1v2sboic2atwqdmgjz6su862ycpuo2z0ko6rr0tbpsn50e9ai59zz92kxk0thyqba6:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 25, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.8467192947171094, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9050990789318509, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9269341155032782, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.9206686019056123, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "86769eac2ab3cf052815f82c71e4088d7ba2864d5f7c488da8c4b319ed046d91", "size": 437, "artifact_path": "wandb-client-artifact://x6fdo4hcuvzmad3vn640kq030ro9rdqjl76dhi33jym2z9ykwi6cx9qah82qriivml4y86ursr8can7q2hhic8geg5vhcnu5gxrjs59iitdaf1eelju2gt3i4m3p7kej/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://syjy03uokkqjx1dpecdvfk5fgbcgws7jmzkrk3uts5t5y8i76wthq9adzl71f36fhk7luwijw52d0ub43r5cz3wsi1pyejmp09jjssfeaplf3lkon1lj2ayqpcq5oeku:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 336}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.8467192649841309, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.905099093914032, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9269341230392456, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.9206686019897461, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug-internal.log b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a3827c1844adb95774b9365de6c54373c4544f96 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug-internal.log @@ -0,0 +1,868 @@ +2025-01-06 22:28:24,164 INFO StreamThr :2126082 [internal.py:wandb_internal():85] W&B internal server running at pid: 2126082, started at: 2025-01-06 22:28:24.164205 +2025-01-06 22:28:24,165 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:28:24,178 INFO WriterThread:2126082 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/run-guo9e7cv.wandb +2025-01-06 22:28:24,179 DEBUG SenderThread:2126082 [sender.py:send():379] send: header +2025-01-06 22:28:24,190 DEBUG SenderThread:2126082 [sender.py:send():379] send: run +2025-01-06 22:28:24,429 INFO SenderThread:2126082 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files +2025-01-06 22:28:24,429 INFO SenderThread:2126082 [sender.py:_start_run_threads():1188] run started: guo9e7cv with start time 1736220504.164651 +2025-01-06 22:28:24,441 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:28:24,441 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:28:24,489 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:28:24,494 DEBUG HandlerThread:2126082 [system_info.py:__init__():26] System info init +2025-01-06 22:28:24,494 DEBUG HandlerThread:2126082 [system_info.py:__init__():41] System info init done +2025-01-06 22:28:24,494 INFO HandlerThread:2126082 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:28:24,494 INFO SystemMonitor:2126082 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:28:24,495 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:28:24,495 INFO HandlerThread:2126082 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:28:24,496 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:28:24,496 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:28:24,497 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:28:24,497 INFO SystemMonitor:2126082 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:28:24,508 DEBUG HandlerThread:2126082 [system_info.py:probe():152] Probing system +2025-01-06 22:28:24,509 DEBUG HandlerThread:2126082 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_info.py:probe():200] Probing system done +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:28:24.508306', 'startedAt': '2025-01-07T03:28:24.156158', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.232125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4496.734, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.499, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 577.9099960327148}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:28:24,514 INFO HandlerThread:2126082 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:28:24,514 INFO HandlerThread:2126082 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:28:24,514 DEBUG HandlerThread:2126082 [system_info.py:_save_code():44] Saving code +2025-01-06 22:28:24,516 DEBUG HandlerThread:2126082 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:28:24,516 DEBUG HandlerThread:2126082 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:28:24,577 DEBUG HandlerThread:2126082 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:28:24,577 INFO HandlerThread:2126082 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:28:24,580 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:28:24,580 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:28:24,581 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:28:24,582 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:28:24,823 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:28:24,823 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:28:24,824 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/1ecwjq0e-diff.patch +2025-01-06 22:28:24,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:28:24,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:24,825 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:28:24,827 INFO HandlerThread:2126082 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/tb" + save: true +} +control { + relay_id: "697ef185a6b841f0aac33ae375b49a0e" +} +_info { + stream_id: "guo9e7cv" +} + +2025-01-06 22:28:24,828 DEBUG HandlerThread:2126082 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:28:24,841 INFO wandb-upload_0:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/30e3vxz6-wandb-metadata.json +2025-01-06 22:28:24,869 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/a686ncs1-code/FISH/eval_robot.py +2025-01-06 22:28:24,897 DEBUG SenderThread:2126082 [sender.py:send():379] send: telemetry +2025-01-06 22:28:24,897 DEBUG SenderThread:2126082 [sender.py:send():379] send: tbrecord +2025-01-06 22:28:24,898 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:28:24,898 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 with policy live +2025-01-06 22:28:25,080 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/tled2t0t-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-metadata.json +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/diff.patch +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH/eval_robot.py +2025-01-06 22:28:25,431 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH +2025-01-06 22:28:25,432 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code +2025-01-06 22:28:25,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:26,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:27,430 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:27,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:28,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:29,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:29,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:30,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:31,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:32,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:33,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:34,618 DEBUG SenderThread:2126082 [sender.py:send():379] send: config +2025-01-06 22:28:34,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:34,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:35,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:36,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:37,433 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:37,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:38,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:39,434 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:39,823 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:28:39,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:28:39,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:39,917 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:40,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:41,434 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:41,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:42,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:43,435 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:43,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:44,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:45,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:45,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:46,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:47,436 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:47,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:48,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:49,437 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:49,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:50,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:51,262 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:51,438 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:28:51,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:52,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:53,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:54,823 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:28:54,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:28:54,865 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:55,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:56,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:56,917 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:28:57,439 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml +2025-01-06 22:28:57,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:58,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:28:59,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:00,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:01,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:02,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:02,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:03,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:04,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:05,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:06,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:07,825 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:07,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:08,829 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:09,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:09,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:09,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:10,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:11,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:12,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:12,930 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:13,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:14,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:15,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:16,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:17,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:18,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:18,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:19,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:20,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:21,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:22,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:23,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:23,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:24,497 DEBUG SystemMonitor:2126082 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:29:24,498 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:29:24,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:24,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:24,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:25,446 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:25,831 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:26,121 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:29:26,121 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 with policy now +2025-01-06 22:29:26,121 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:29:26,122 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:29:26,122 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:29:26,123 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:29:26,340 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/3yjde5jn-media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media +2025-01-06 22:29:26,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos +2025-01-06 22:29:26,712 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/l7hmfe2f-media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:29:26,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:27,447 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:27,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:28,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:29,135 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:29,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:30,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:31,448 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:31,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:32,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:33,449 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:33,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:34,489 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:34,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:35,450 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:35,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:36,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:37,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:38,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:39,451 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:29:39,690 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/ekc0u904-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:29:39,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:39,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:39,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:39,977 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:40,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:41,452 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:41,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:42,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:43,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:44,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:45,453 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:29:45,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:45,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:46,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:47,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:48,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:49,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:50,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:50,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:51,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:52,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:53,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:54,498 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:29:54,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:29:54,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:29:54,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:55,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:55,973 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:29:56,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:57,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:58,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:29:59,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:00,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:01,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:01,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:02,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:03,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:04,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:05,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:06,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:06,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:07,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:08,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:09,824 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:09,824 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:09,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:10,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:11,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:11,889 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:12,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:13,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:14,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:15,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:16,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:17,463 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:17,817 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:17,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:18,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:19,852 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:19,905 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:30:19,905 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:30:19,905 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 with policy now +2025-01-06 22:30:19,906 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:30:19,906 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:30:19,910 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:30:20,145 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/xf1clufe-media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:30:20,464 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:30:20,464 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:30:20,465 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:30:20,755 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/ghalz7oj-media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:30:20,856 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:21,465 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:21,852 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:22,852 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:22,921 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:23,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:24,499 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:30:24,827 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:24,827 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:24,869 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:25,466 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:25,854 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:26,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:27,466 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:27,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:28,093 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:28,853 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:29,467 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:30:29,853 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:30,250 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:30:30,250 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:30:30,251 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:30:30,468 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:30:30,856 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:31,468 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:31,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:32,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:33,854 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:34,071 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:34,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:35,469 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:35,855 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:36,856 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:37,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:38,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:39,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:39,827 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:39,873 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:39,925 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:40,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:41,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:42,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:43,473 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:43,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:44,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:45,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:45,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:46,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:47,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:48,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:49,475 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:49,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:50,835 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:50,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:51,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:52,858 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:53,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:54,499 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:30:54,830 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:30:54,830 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:30:54,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:55,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:55,875 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:30:56,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:57,478 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:30:57,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:58,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:30:59,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:00,859 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:01,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:01,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:02,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:03,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:04,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:05,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:06,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:07,836 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:07,861 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:08,872 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:09,482 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:09,730 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:31:09,730 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 with policy now +2025-01-06 22:31:09,731 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:31:09,732 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:31:09,732 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:31:09,738 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:31:09,826 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:09,827 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:09,872 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:09,969 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/p1u3gqbg-media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:31:10,482 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:31:10,483 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:31:10,483 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:31:10,760 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/jflcmz2z-media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:31:10,878 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:11,483 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:11,876 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:12,875 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:12,960 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:13,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:14,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:15,484 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:15,876 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:16,880 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:17,485 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:17,877 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:18,321 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:18,882 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:19,880 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:20,885 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:21,884 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:22,883 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:23,844 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:23,889 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:24,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:31:24,839 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:24,839 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:24,884 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:25,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:26,889 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:27,488 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:31:27,728 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/06xwrsc8-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:31:27,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:28,096 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:31:28,097 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:31:28,097 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:31:28,489 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:31:28,899 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:29,429 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:29,489 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:29,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:30,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:31,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:32,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:33,490 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:33,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:34,841 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:34,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:35,491 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:31:35,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:36,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:37,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:38,891 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:39,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:39,829 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:39,891 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:39,951 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:40,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:41,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:42,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:43,892 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:44,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:45,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:45,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:46,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:47,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:48,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:49,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:50,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:50,893 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:51,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:52,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:53,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:54,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:31:54,828 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:31:54,828 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:31:54,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:55,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:55,942 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:31:56,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:57,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:58,894 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:31:59,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:00,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:01,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:01,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:02,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:03,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:04,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:05,895 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:06,842 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:06,896 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:07,896 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:08,896 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:08,992 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:32:08,994 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:32:08,997 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 with policy now +2025-01-06 22:32:08,998 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:32:08,998 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:32:09,021 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:32:09,238 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/45ce5fpw-media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:32:09,503 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:09,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:32:09,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:32:09,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:32:09,792 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/dv1m6pkh-media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:32:09,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:09,837 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:09,899 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:10,910 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:11,504 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:11,901 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:11,910 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:12,913 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:13,505 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:13,913 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:14,919 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:15,505 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:15,911 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:16,915 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:17,477 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:17,506 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:17,920 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:18,929 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:19,926 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:20,924 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:21,925 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:22,853 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:22,932 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:23,934 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:24,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:32:24,832 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:24,833 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:24,929 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:25,928 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:26,928 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:27,914 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:27,929 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:28,928 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:29,930 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:30,932 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:31,931 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:32,936 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:33,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:33,936 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:34,937 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:35,937 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:36,939 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:37,944 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:38,860 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:38,945 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:39,837 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:39,837 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:39,943 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:40,976 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:41,961 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:42,961 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:43,962 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:43,972 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:44,968 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:45,973 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:46,964 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:47,979 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:48,974 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:49,854 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:49,978 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:50,978 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:51,975 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:52,978 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:53,981 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:54,196 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:32:54,197 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:32:54,198 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:32:54,501 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:32:54,518 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:32:54,519 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:32:54,771 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/u5kzdn73-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:32:54,840 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:32:54,840 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:32:54,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:54,995 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:32:55,519 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:55,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:56,981 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:57,519 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:57,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:58,993 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:32:59,520 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:32:59,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:00,575 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:00,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:01,521 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:01,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:02,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:03,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:04,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:05,849 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:05,982 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:06,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:07,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:08,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:09,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:09,834 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:09,983 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:10,984 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:10,985 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:11,984 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:12,985 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:13,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:14,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:15,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:16,851 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:16,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:17,986 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:18,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:19,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:20,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:21,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:22,851 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:22,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:23,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:24,501 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:33:24,833 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:24,834 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:24,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:25,987 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:26,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:27,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:28,437 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:28,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:29,530 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:29,988 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:30,990 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:31,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:32,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:33,851 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:33,989 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:34,990 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:36,002 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:36,286 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:33:36,286 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 with policy now +2025-01-06 22:33:36,292 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:33:36,292 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:36,292 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:36,292 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:36,520 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/r6avyvi4-media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:33:36,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:36,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:33:36,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval +2025-01-06 22:33:36,804 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/5ock1lma-media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:33:36,995 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:37,533 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:38,000 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:38,995 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:39,343 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:39,838 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:39,838 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:39,999 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:41,000 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:41,534 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:42,000 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:43,007 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:44,012 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:44,806 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:45,013 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:45,536 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:46,020 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:47,019 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:48,017 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:49,020 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:49,857 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:50,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:51,026 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:52,031 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:52,270 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:52,270 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:52,272 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:52,538 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:52,554 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:33:52,568 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:33:52,568 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:33:52,761 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:33:53,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:53,039 INFO wandb-upload_2:2126082 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptuu9c131 +2025-01-06 22:33:53,253 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:33:53,263 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:33:53,479 INFO SenderThread:2126082 [sender.py:send_request_log_artifact():1525] logged artifact run-guo9e7cv-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTAwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0ODU0MA==', 'latestArtifact': None}} +2025-01-06 22:33:53,479 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:33:53,479 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 22:33:53,479 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:53,479 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,500 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,500 DEBUG SenderThread:2126082 [sender.py:send():379] send: telemetry +2025-01-06 22:33:53,500 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,501 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,501 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,525 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,526 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media +2025-01-06 22:33:53,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table +2025-01-06 22:33:53,540 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval +2025-01-06 22:33:53,546 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,547 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,563 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,564 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,564 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,564 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,608 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,609 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,630 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,630 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,631 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,631 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,653 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,654 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,654 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,654 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:53,655 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/0j382wkx-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:33:53,675 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:53,676 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:33:53,773 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/c4lx72ro-media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:33:53,904 INFO wandb-upload_0:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/a220e7co-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:33:54,027 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:54,050 INFO wandb-upload_1:2126082 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyjgfxxn7 +2025-01-06 22:33:54,407 INFO SenderThread:2126082 [sender.py:send_request_log_artifact():1525] logged artifact run-guo9e7cv-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTAzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0ODU0Mg==', 'latestArtifact': None}} +2025-01-06 22:33:54,407 DEBUG SenderThread:2126082 [sender.py:send():379] send: files +2025-01-06 22:33:54,407 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json with policy now +2025-01-06 22:33:54,407 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:33:54,408 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:33:54,408 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:33:54,502 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:33:54,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:33:54,676 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/wskxxezf-media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:33:54,834 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:33:54,834 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:33:54,890 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:33:55,030 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:55,539 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:33:56,027 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:57,028 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:58,028 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:33:59,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:00,029 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:00,630 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:00,677 DEBUG SenderThread:2126082 [sender.py:send():379] send: exit +2025-01-06 22:34:00,677 INFO SenderThread:2126082 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:send_exit():588] handling runtime: 336 +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:send_exit():594] send defer +2025-01-06 22:34:00,678 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,678 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:34:00,678 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,678 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:34:00,679 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:34:00,679 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:34:00,679 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,679 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:34:00,679 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:34:00,679 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:34:00,679 DEBUG SystemMonitor:2126082 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:34:00,679 DEBUG SystemMonitor:2126082 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:34:00,679 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:34:00,680 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:34:00,732 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:34:00,733 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:34:00,733 INFO HandlerThread:2126082 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:34:00,733 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,733 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:34:00,733 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:34:00,733 DEBUG SenderThread:2126082 [sender.py:send():379] send: stats +2025-01-06 22:34:00,733 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,734 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:34:00,734 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:00,734 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:34:00,734 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:34:00,734 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:00,734 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:34:01,542 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:34:01,542 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:34:07,276 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:07,276 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:07,276 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:34:07,277 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,277 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:34:07,277 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:34:07,278 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:07,278 DEBUG SenderThread:2126082 [sender.py:send():379] send: history +2025-01-06 22:34:07,278 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:34:07,278 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:07,278 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:34:07,279 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:34:07,279 DEBUG SenderThread:2126082 [sender.py:send():379] send: summary +2025-01-06 22:34:07,282 INFO SenderThread:2126082 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:34:07,283 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,283 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:34:07,283 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:34:07,283 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:07,283 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:34:07,283 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,283 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:34:07,287 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:07,391 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:34:07,391 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:07,391 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:34:07,392 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:07,392 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:34:07,544 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml +2025-01-06 22:34:07,544 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:34:07,678 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:09,241 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:34:09,241 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:09,241 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:09,241 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:34:09,242 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:09,242 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:34:09,242 INFO SenderThread:2126082 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:34:09,244 INFO SenderThread:2126082 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:34:09,246 INFO SenderThread:2126082 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:34:09,261 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:34:09,261 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:09,261 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:34:09,262 DEBUG SenderThread:2126082 [sender.py:send():379] send: artifact +2025-01-06 22:34:09,545 INFO Thread-12 :2126082 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:34:09,679 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:09,915 INFO wandb-upload_2:2126082 [upload_job.py:push():85] Skipped uploading /tmp/tmp_ymzf5nw/wandb-job.json +2025-01-06 22:34:09,916 INFO wandb-upload_0:2126082 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4m9y5tu9 +2025-01-06 22:34:10,037 INFO wandb-upload_1:2126082 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpsg96ts72 +2025-01-06 22:34:10,815 INFO SenderThread:2126082 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:34:10,815 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:10,816 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:34:10,816 INFO SenderThread:2126082 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:34:11,546 INFO SenderThread:2126082 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml config.yaml +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json wandb-summary.json +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt requirements.txt +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:34:11,547 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log output.log +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/diff.patch diff.patch +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 media/videos/eval/1_eval_1_7fe2f42bb4fd94a8c29e.mp4 +2025-01-06 22:34:11,552 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 media/videos/eval/3_eval_3_81ffc5d904861fc29e86.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 media/videos/eval/2_eval_2_8707cf4f2647980f40ba.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 media/videos/eval/4_eval_4_8973ebd698ddc1cd5e38.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 media/videos/eval/0_eval_0_decc75133e15da63fd7d.mp4 +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json media/table/eval/total_success_rate_ci_6_86769eac2ab3cf052815.table.json +2025-01-06 22:34:11,553 INFO SenderThread:2126082 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:34:11,554 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:34:11,554 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:11,555 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:11,555 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:34:11,556 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:11,556 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:34:11,556 INFO SenderThread:2126082 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:34:11,680 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:11,680 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:11,718 INFO wandb-upload_2:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/config.yaml +2025-01-06 22:34:11,739 INFO wandb-upload_0:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/wandb-summary.json +2025-01-06 22:34:11,744 INFO wandb-upload_1:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/requirements.txt +2025-01-06 22:34:11,778 INFO wandb-upload_4:2126082 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/files/output.log +2025-01-06 22:34:11,832 INFO wandb-upload_3:2126082 [upload_job.py:push():130] Uploaded file /tmp/tmpec9c_acbwandb/vp03xi33-events.out.tfevents.1736220504.leonmkim-ROG-Strix-G15CS-G15CS.2125886.0 +2025-01-06 22:34:12,032 INFO Thread-11 (_thread_body):2126082 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:34:12,032 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,032 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:34:12,032 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,033 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:34:12,033 INFO SenderThread:2126082 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:34:12,034 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:34:12,034 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,034 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:34:12,035 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,035 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:34:12,035 INFO SenderThread:2126082 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:34:12,099 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,099 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:34:12,099 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:34:12,099 INFO SenderThread:2126082 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:34:12,099 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:34:12,100 INFO HandlerThread:2126082 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send():379] send: final +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send():379] send: footer +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: defer +2025-01-06 22:34:12,100 INFO SenderThread:2126082 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:34:12,100 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:12,100 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:12,101 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:34:12,101 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:34:12,102 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:34:12,103 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:34:12,103 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:34:12,104 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:12,148 DEBUG SenderThread:2126082 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:34:12,149 INFO MainThread:2126082 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:34:12,149 INFO MainThread:2126082 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:34:12,149 INFO MainThread:2126082 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:34:12,150 DEBUG HandlerThread:2126082 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:34:12,150 INFO HandlerThread:2126082 [handler.py:finish():882] shutting down handler +2025-01-06 22:34:13,104 INFO WriterThread:2126082 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/run-guo9e7cv.wandb +2025-01-06 22:34:13,149 INFO SenderThread:2126082 [sender.py:finish():1615] shutting down sender +2025-01-06 22:34:13,149 INFO SenderThread:2126082 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:34:13,149 INFO SenderThread:2126082 [file_pusher.py:join():175] waiting for file pusher diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug.log b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0c40cadc2dee5989a88495ff31e229133cd993c4 --- /dev/null +++ b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Configure stats pid to 2125886 +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/settings +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug.log +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/logs/debug-internal.log +2025-01-06 22:28:24,158 INFO MainThread:2125886 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:28:24,159 INFO MainThread:2125886 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv'} +2025-01-06 22:28:24,159 INFO MainThread:2125886 [wandb_init.py:init():619] starting backend +2025-01-06 22:28:24,159 INFO MainThread:2125886 [wandb_init.py:init():623] setting up manager +2025-01-06 22:28:24,163 INFO MainThread:2125886 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:28:24,164 INFO MainThread:2125886 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:28:24,177 INFO MainThread:2125886 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:28:24,185 INFO MainThread:2125886 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:28:24,440 INFO MainThread:2125886 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:28:24,479 INFO MainThread:2125886 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:28:24,479 INFO MainThread:2125886 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:28:24,823 INFO MainThread:2125886 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:28:24,824 INFO MainThread:2125886 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:28:24,824 INFO MainThread:2125886 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:28:24,824 INFO MainThread:2125886 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:28:24,825 INFO MainThread:2125886 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:28:24,825 INFO MainThread:2125886 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/guo9e7cv/tb, True +2025-01-06 22:28:34,618 INFO MainThread:2125886 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:34:13,150 WARNING MsgRouterThr:2125886 [router.py:message_loop():77] message_loop has been closed diff --git a/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/run-guo9e7cv.wandb b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/run-guo9e7cv.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e42afa791bd6a1a06188378af30aaf5c29ca9213 Binary files /dev/null and b/guo9e7cv/wandb/run-20250106_222824-guo9e7cv/run-guo9e7cv.wandb differ diff --git a/h2bqbukm/episode_rosbags/cam_tf_world.npy b/h2bqbukm/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/h2bqbukm/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/h2bqbukm/episode_rosbags/depth_K.npy b/h2bqbukm/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/h2bqbukm/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/h2bqbukm/eval_robot.log b/h2bqbukm/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..34c8fa3461ad7ec88657cadcebb4b74e250b6b4c --- /dev/null +++ b/h2bqbukm/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:49:39,094][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:49:39,097][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/hcdwvdao/.hydra/config.yaml b/hcdwvdao/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/hcdwvdao/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/hcdwvdao/.hydra/hydra.yaml b/hcdwvdao/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..139e3ccd14baaddaa1a7c0f448fee1bae4b7c4a9 --- /dev/null +++ b/hcdwvdao/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/hcdwvdao/.hydra/overrides.yaml b/hcdwvdao/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/hcdwvdao/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/hcdwvdao/episode_rosbags/aligned_depth_to_color_K.npy b/hcdwvdao/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hcdwvdao/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hcdwvdao/episode_rosbags/cam_tf_world.npy b/hcdwvdao/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/hcdwvdao/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/hcdwvdao/episode_rosbags/color_K.npy b/hcdwvdao/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hcdwvdao/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hcdwvdao/episode_rosbags/depth_K.npy b/hcdwvdao/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/hcdwvdao/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/hcdwvdao/eval_robot.log b/hcdwvdao/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..64800d7031d4b3997730e78d65a54267e9773008 --- /dev/null +++ b/hcdwvdao/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 18:06:13,605][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 18:06:13,608][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/hcdwvdao/eval_video/0_eval.mp4 b/hcdwvdao/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0535b498557b89082849055550d002e1ade24b7e --- /dev/null +++ b/hcdwvdao/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0912050a1ad9dbe66ae9cd19192f72af3835002662afcb6109b27cd1a30a67d +size 1442898 diff --git a/hcdwvdao/eval_video/1_eval.mp4 b/hcdwvdao/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..494ca4f0ff754d032617a8011112fff3ea4d03ca --- /dev/null +++ b/hcdwvdao/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d770d2c2408f04d7abb1e233b4ce8b6adc12ae2851e1e74d9b31b4d95ca67fe6 +size 1454280 diff --git a/hcdwvdao/eval_video/2_eval.mp4 b/hcdwvdao/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..553374901c74f1b09bc75a5f34dc429895f59cdb --- /dev/null +++ b/hcdwvdao/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b10b1d7fc921011b04f06948c7c2f9bf565d338709580df152bd2ff0e1090a +size 1470576 diff --git a/hcdwvdao/eval_video/3_eval.mp4 b/hcdwvdao/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..03a09a314e96e160318c9b9b5a690ed78085d936 --- /dev/null +++ b/hcdwvdao/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4b0a7d2f5d2484c5d273f279f87160b09da9676192ce6ae529fcdc730947e2b +size 1472690 diff --git a/hcdwvdao/eval_video/4_eval.mp4 b/hcdwvdao/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b503d533295d5b6831952f4c1e8280742890040 --- /dev/null +++ b/hcdwvdao/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2805344fc05b94c9bd94388782d346624b7dbcd9616124458c77d89ddd1f0846 +size 1483706 diff --git a/hcdwvdao/tb/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 b/hcdwvdao/tb/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 new file mode 100644 index 0000000000000000000000000000000000000000..2c495d0e0c4fe00fee524625bb3c0a90d32dd3b4 --- /dev/null +++ b/hcdwvdao/tb/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec034bcc3e2900a8332ce1fd4d127b1ee42298752e3dd08cdee7ab71eff06795 +size 6539 diff --git a/hcdwvdao/wandb/debug-internal.log b/hcdwvdao/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..516a9a0d6e150a7ccac257410e473aa2b0cee700 --- /dev/null +++ b/hcdwvdao/wandb/debug-internal.log @@ -0,0 +1,800 @@ +2025-01-07 18:06:19,355 INFO StreamThr :2207962 [internal.py:wandb_internal():85] W&B internal server running at pid: 2207962, started at: 2025-01-07 18:06:19.355271 +2025-01-07 18:06:19,356 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:06:19,368 INFO WriterThread:2207962 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/run-hcdwvdao.wandb +2025-01-07 18:06:19,368 DEBUG SenderThread:2207962 [sender.py:send():379] send: header +2025-01-07 18:06:19,375 DEBUG SenderThread:2207962 [sender.py:send():379] send: run +2025-01-07 18:06:19,630 INFO SenderThread:2207962 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files +2025-01-07 18:06:19,631 INFO SenderThread:2207962 [sender.py:_start_run_threads():1188] run started: hcdwvdao with start time 1736291179.357482 +2025-01-07 18:06:19,646 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:06:19,647 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:06:19,720 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:06:19,725 DEBUG HandlerThread:2207962 [system_info.py:__init__():26] System info init +2025-01-07 18:06:19,725 DEBUG HandlerThread:2207962 [system_info.py:__init__():41] System info init done +2025-01-07 18:06:19,725 INFO HandlerThread:2207962 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:06:19,725 INFO SystemMonitor:2207962 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:06:19,725 INFO HandlerThread:2207962 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:06:19,726 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:06:19,727 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:06:19,727 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:06:19,728 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:06:19,728 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:06:19,739 DEBUG HandlerThread:2207962 [system_info.py:probe():152] Probing system +2025-01-07 18:06:19,740 DEBUG HandlerThread:2207962 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:06:19,745 DEBUG HandlerThread:2207962 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:06:19,745 DEBUG HandlerThread:2207962 [system_info.py:probe():200] Probing system done +2025-01-07 18:06:19,745 DEBUG HandlerThread:2207962 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:06:19.739320', 'startedAt': '2025-01-07T23:06:19.347690', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3566.71675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4502.251, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4509.34, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 802.8848648071289}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:06:19,746 INFO HandlerThread:2207962 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:06:19,746 INFO HandlerThread:2207962 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:06:19,746 DEBUG HandlerThread:2207962 [system_info.py:_save_code():44] Saving code +2025-01-07 18:06:19,747 DEBUG HandlerThread:2207962 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:06:19,747 DEBUG HandlerThread:2207962 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:06:19,809 DEBUG HandlerThread:2207962 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:06:19,810 INFO HandlerThread:2207962 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:06:19,813 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:06:19,813 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:06:19,813 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:06:19,813 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 18:06:20,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:06:20,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:06:20,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:06:20,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:06:20,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:20,057 INFO HandlerThread:2207962 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/tb" + save: true +} +control { + relay_id: "55d9574faf0842428b21c40159837d41" +} +_info { + stream_id: "hcdwvdao" +} + +2025-01-07 18:06:20,057 DEBUG HandlerThread:2207962 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:06:20,068 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/eifv9h5e-code/FISH/eval_robot.py +2025-01-07 18:06:20,082 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/3cvz4p6z-wandb-metadata.json +2025-01-07 18:06:20,132 DEBUG SenderThread:2207962 [sender.py:send():379] send: telemetry +2025-01-07 18:06:20,132 DEBUG SenderThread:2207962 [sender.py:send():379] send: tbrecord +2025-01-07 18:06:20,132 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:06:20,133 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 with policy live +2025-01-07 18:06:20,142 INFO wandb-upload_2:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/3i0p729g-diff.patch +2025-01-07 18:06:20,321 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/h3heor1y-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:06:20,632 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:20,632 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt +2025-01-07 18:06:20,632 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH/eval_robot.py +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-metadata.json +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/diff.patch +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code +2025-01-07 18:06:21,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:22,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:22,631 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:23,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:23,772 DEBUG SenderThread:2207962 [sender.py:send():379] send: config +2025-01-07 18:06:24,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:24,785 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:25,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:26,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:26,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:27,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:28,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:28,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:29,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:30,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:30,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:30,634 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:31,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:32,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:32,634 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:33,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:34,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:35,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:06:35,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:06:35,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:35,115 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:36,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:37,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:38,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:39,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:40,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:40,637 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:41,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:41,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:42,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:42,637 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:43,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:44,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:44,638 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:45,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:46,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:46,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:47,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:48,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:49,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:50,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:06:50,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:06:50,101 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:51,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:51,169 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:51,640 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml +2025-01-07 18:06:52,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:53,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:54,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:55,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:56,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:57,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:57,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:58,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:59,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:00,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:01,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:02,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:02,497 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:02,642 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:03,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:04,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:04,643 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:05,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:05,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:05,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:06,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:07,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:08,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:08,153 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:09,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:10,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:11,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:12,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:13,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:14,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:14,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:15,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:16,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:17,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:17,802 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:07:17,802 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 with policy now +2025-01-07 18:07:17,803 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:07:17,806 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:07:17,806 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:07:17,807 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:07:18,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:18,067 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/xfkr53fc-media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media +2025-01-07 18:07:18,648 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos +2025-01-07 18:07:18,648 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:07:18,948 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/xmoxyftm-media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:07:19,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:19,728 DEBUG SystemMonitor:2207962 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:07:19,729 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:07:19,729 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:20,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:20,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:20,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:20,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:21,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:22,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:22,648 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:23,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:24,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:25,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:25,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:26,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:26,649 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:27,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:28,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:28,650 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:07:28,945 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/mwkiq0be-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:07:29,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:30,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:30,651 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:31,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:31,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:32,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:33,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:34,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:34,652 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:35,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:35,059 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:35,101 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:36,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:36,218 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:37,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:38,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:39,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:40,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:41,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:42,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:42,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:43,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:44,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:45,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:46,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:47,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:47,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:48,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:49,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:49,729 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:07:50,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:50,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:50,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:51,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:52,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:52,214 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:53,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:54,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:55,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:56,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:57,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:58,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:58,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:59,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:00,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:01,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:02,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:03,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:03,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:04,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:05,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:05,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:05,101 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:06,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:07,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:08,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:08,423 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:08,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:08,676 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:08:08,676 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 with policy now +2025-01-07 18:08:08,679 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:08:08,679 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:08:08,679 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:08:08,683 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:08:08,951 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/ejiokp1q-media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:08:09,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:09,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:08:09,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:08:09,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:08:09,984 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/qmp57b2j-media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:08:10,075 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:10,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:11,078 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:12,080 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:13,080 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:13,453 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:14,078 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:14,665 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:15,080 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:16,079 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:16,665 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:17,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:18,088 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:18,666 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:08:19,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:19,413 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:19,494 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:08:19,494 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:08:19,495 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:08:19,667 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:08:19,730 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:08:20,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:20,057 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:20,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:20,667 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:21,086 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:22,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:23,082 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:24,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:24,483 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:24,668 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:25,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:26,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:27,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:28,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:29,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:30,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:30,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:31,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:32,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:33,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:34,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:35,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:35,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:35,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:35,149 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:36,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:37,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:38,277 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:39,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:40,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:41,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:41,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:42,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:42,680 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:43,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:44,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:45,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:46,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:46,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:47,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:48,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:49,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:49,730 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:08:50,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:50,056 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:50,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:51,114 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:51,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:52,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:53,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:54,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:55,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:56,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:57,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:57,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:58,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:59,217 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:08:59,217 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 with policy now +2025-01-07 18:08:59,220 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:08:59,220 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:08:59,220 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:08:59,226 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:08:59,274 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:59,473 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/400suiaq-media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:08:59,687 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:08:59,688 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:08:59,688 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:08:59,964 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/7gfiypmj-media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:09:00,281 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:00,688 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:01,276 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:02,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:02,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:03,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:04,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:04,689 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:05,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:05,058 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:05,277 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:06,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:06,690 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:07,277 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:07,777 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:08,279 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:09,282 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:10,286 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:11,282 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:12,284 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:13,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:13,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:14,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:15,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:16,289 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:17,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:18,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:18,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:18,347 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:09:18,347 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:09:18,348 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:09:18,694 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:09:18,694 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:09:18,955 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/neeeybwj-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:09:19,321 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:19,731 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:09:20,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:20,057 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:20,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:20,694 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:21,295 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:22,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:23,280 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:23,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:24,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:24,696 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:25,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:26,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:27,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:28,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:29,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:29,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:30,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:31,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:32,289 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:33,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:34,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:34,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:35,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:35,057 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:35,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:36,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:37,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:38,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:39,118 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:39,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:40,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:41,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:42,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:43,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:44,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:45,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:45,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:46,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:47,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:48,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:49,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:49,731 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:09:50,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:50,056 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:50,211 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:50,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:51,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:52,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:53,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:54,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:55,294 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:56,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:56,294 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:57,295 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:58,295 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:58,999 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:09:58,999 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 with policy now +2025-01-07 18:09:59,002 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:09:59,002 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:09:59,003 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:09:59,009 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:09:59,262 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/5ymy2z1j-media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:09:59,296 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:59,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:09:59,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:09:59,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:09:59,989 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/6l876urn-media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:10:00,302 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:00,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:01,075 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:01,300 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:02,299 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:03,301 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:04,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:04,709 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:05,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:05,063 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:05,314 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:06,303 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:06,573 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:06,710 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:07,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:08,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:09,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:09,711 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:10:10,302 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:10,412 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:10:10,413 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:10:10,413 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:10:10,711 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:10:10,712 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:11,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:11,693 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:12,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:12,712 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:13,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:14,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:14,713 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:15,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:16,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:16,714 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:17,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:17,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:18,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:19,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:19,732 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:10:20,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:20,059 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:20,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:21,307 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:22,170 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:22,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:23,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:24,307 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:25,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:26,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:27,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:28,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:28,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:29,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:30,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:31,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:32,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:33,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:33,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:34,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:35,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:35,059 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:35,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:36,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:37,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:38,143 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:38,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:39,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:40,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:41,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:42,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:43,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:44,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:44,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:45,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:46,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:47,312 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:48,312 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:49,312 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:49,391 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:49,732 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:10:50,074 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:50,074 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:50,322 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:50,429 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:10:50,429 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 with policy now +2025-01-07 18:10:50,435 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:10:50,435 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:10:50,435 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:10:50,443 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:10:50,737 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/ioczzw4e-media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:10:51,009 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/dmn7kbli-media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:10:51,324 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:52,331 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:52,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:53,324 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:54,327 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:55,079 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:55,326 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:56,330 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:56,729 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:57,332 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:58,329 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:58,729 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:59,329 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:00,090 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:00,336 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:01,338 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:02,333 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:03,338 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:04,340 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:05,065 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:11:05,065 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:11:05,189 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:05,342 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:06,341 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:07,336 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:08,340 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:09,348 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:09,490 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:09,490 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:09,490 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:09,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:09,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:09,994 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/xj7zdd1l-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:10,313 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:11:10,313 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:11:10,357 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:10,553 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:11:10,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:10,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:10,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media +2025-01-07 18:11:10,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval +2025-01-07 18:11:10,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table +2025-01-07 18:11:10,820 INFO wandb-upload_1:2207962 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn0uzkz_a +2025-01-07 18:11:10,966 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:11:10,980 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:11:10,990 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/dc8zpfxj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:11,194 INFO SenderThread:2207962 [sender.py:send_request_log_artifact():1525] logged artifact run-hcdwvdao-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA2NjkyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc0MzgzNQ==', 'latestArtifact': None}} +2025-01-07 18:11:11,194 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:11:11,194 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 18:11:11,195 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:11,195 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:11,195 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,196 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,196 DEBUG SenderThread:2207962 [sender.py:send():379] send: telemetry +2025-01-07 18:11:11,196 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,200 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,200 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,200 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,200 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,200 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,201 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,201 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,201 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,202 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,202 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,202 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,202 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,203 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,203 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,203 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,203 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,204 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,204 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,204 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,204 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,205 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,205 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:11:11,338 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:11,466 INFO wandb-upload_2:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/gq1xcm8x-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:11,587 INFO wandb-upload_0:2207962 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa5g1zu2x +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval +2025-01-07 18:11:11,962 INFO SenderThread:2207962 [sender.py:send_request_log_artifact():1525] logged artifact run-hcdwvdao-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA2NjkzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc0MzgzNg==', 'latestArtifact': None}} +2025-01-07 18:11:11,962 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:11:11,962 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json with policy now +2025-01-07 18:11:11,962 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:11,963 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,963 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,963 DEBUG SenderThread:2207962 [sender.py:send():379] send: exit +2025-01-07 18:11:11,963 INFO SenderThread:2207962 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:11:11,963 INFO SenderThread:2207962 [sender.py:send_exit():588] handling runtime: 291 +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:send_exit():594] send defer +2025-01-07 18:11:11,964 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:11,964 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:11:11,964 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:11:11,964 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:11,964 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:11:11,964 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:11:11,965 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:11:11,965 DEBUG SystemMonitor:2207962 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:11:11,965 DEBUG SystemMonitor:2207962 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:11:12,017 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:11:12,018 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:11:12,018 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:11:12,018 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:12,018 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:11:12,018 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:11:12,018 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:11:12,019 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:12,019 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:11:12,019 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:12,019 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:11:12,019 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:11:12,019 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:12,019 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:11:12,147 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/pta6s1go-media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:12,404 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/rmqd7loz-media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:12,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:12,735 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:14,735 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:18,492 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:18,492 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:18,492 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:11:18,493 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,493 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:11:18,493 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:11:18,494 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:18,494 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:18,494 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:18,494 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:18,495 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:11:18,496 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:18,496 DEBUG SenderThread:2207962 [sender.py:send():379] send: summary +2025-01-07 18:11:18,500 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:18,500 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,500 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:11:18,500 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:11:18,500 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:18,501 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:11:18,501 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,501 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:11:18,504 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:18,633 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:11:18,633 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:18,633 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:11:18,634 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,634 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:11:18,737 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:18,737 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml +2025-01-07 18:11:19,459 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:20,425 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:11:20,425 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:20,425 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:20,425 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:11:20,426 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:20,426 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:11:20,426 INFO SenderThread:2207962 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:11:20,428 INFO SenderThread:2207962 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:11:20,430 INFO SenderThread:2207962 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:11:20,445 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:11:20,445 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:20,445 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:11:20,445 DEBUG SenderThread:2207962 [sender.py:send():379] send: artifact +2025-01-07 18:11:20,459 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:20,670 INFO SenderThread:2207962 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:11:20,671 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:20,671 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:11:20,671 INFO SenderThread:2207962 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:11:20,738 INFO SenderThread:2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:20,738 INFO SenderThread:2207962 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files +2025-01-07 18:11:20,738 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml config.yaml +2025-01-07 18:11:20,739 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json wandb-summary.json +2025-01-07 18:11:20,740 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:20,740 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt requirements.txt +2025-01-07 18:11:20,740 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:11:20,741 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log output.log +2025-01-07 18:11:20,741 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/diff.patch diff.patch +2025-01-07 18:11:20,742 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:11:20,742 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:11:20,742 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:11:20,743 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:20,746 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:20,746 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:11:20,748 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:20,748 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:11:20,748 INFO SenderThread:2207962 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:11:20,933 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml +2025-01-07 18:11:20,934 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:20,984 INFO wandb-upload_4:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:20,984 INFO wandb-upload_3:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt +2025-01-07 18:11:21,035 INFO wandb-upload_2:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/n4hex9g0-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:21,235 INFO Thread-11 (_thread_body):2207962 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:11:21,235 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,235 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:11:21,236 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,236 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:11:21,236 INFO SenderThread:2207962 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:11:21,237 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:11:21,237 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,237 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:11:21,237 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,237 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:11:21,237 INFO SenderThread:2207962 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:11:21,460 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:21,561 INFO SenderThread:2207962 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:11:21,561 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:11:21,561 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:21,561 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,561 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:11:21,561 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,562 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:11:21,562 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:11:21,562 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,562 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:11:21,562 DEBUG SenderThread:2207962 [sender.py:send():379] send: final +2025-01-07 18:11:21,562 DEBUG SenderThread:2207962 [sender.py:send():379] send: footer +2025-01-07 18:11:21,563 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,563 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:11:21,563 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:21,563 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:21,564 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:11:21,564 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:21,564 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:11:21,565 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:11:21,565 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:11:21,565 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:21,618 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:21,618 INFO MainThread:2207962 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:11:21,619 INFO MainThread:2207962 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:11:21,619 INFO MainThread:2207962 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:11:21,619 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:11:21,619 INFO HandlerThread:2207962 [handler.py:finish():882] shutting down handler +2025-01-07 18:11:22,564 INFO WriterThread:2207962 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/run-hcdwvdao.wandb +2025-01-07 18:11:22,618 INFO SenderThread:2207962 [sender.py:finish():1615] shutting down sender +2025-01-07 18:11:22,618 INFO SenderThread:2207962 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:11:22,618 INFO SenderThread:2207962 [file_pusher.py:join():175] waiting for file pusher diff --git a/hcdwvdao/wandb/debug.log b/hcdwvdao/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..be7b335e02f5079935ac264b8f3bc4de68f2fdd8 --- /dev/null +++ b/hcdwvdao/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Configure stats pid to 2207769 +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/settings +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug.log +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug-internal.log +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao'} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():619] starting backend +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():623] setting up manager +2025-01-07 18:06:19,354 INFO MainThread:2207769 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:06:19,356 INFO MainThread:2207769 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:06:19,367 INFO MainThread:2207769 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:06:19,373 INFO MainThread:2207769 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:06:19,646 INFO MainThread:2207769 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:06:19,714 INFO MainThread:2207769 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:06:19,714 INFO MainThread:2207769 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:06:20,054 INFO MainThread:2207769 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:06:20,054 INFO MainThread:2207769 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:06:20,055 INFO MainThread:2207769 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:06:20,055 INFO MainThread:2207769 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:06:20,056 INFO MainThread:2207769 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:06:20,057 INFO MainThread:2207769 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/tb, True +2025-01-07 18:06:23,771 INFO MainThread:2207769 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:11:22,620 WARNING MsgRouterThr:2207769 [router.py:message_loop():77] message_loop has been closed diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH/eval_robot.py b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1c3a27bad35c2be09df72aa3fa0a616a75b0b6e --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1003_0' +wandb_notes: + desc: null + value: 1003_0_req_1364_0restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736291179 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/diff.patch b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..4697b4ebb6e44a4ea64b46b459fc2cf4999dc3f0 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/diff.patch @@ -0,0 +1,16 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..1643b06 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,9 +138,9 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +-# wandb_run_id: '1003_0' ++wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e33018f32a2588b1d4ac684dea9349a63d80705e --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[38, 45, 0.8444444444444444, 0.6862424909696025, 0.9418053179095713, 0.7218401282582063, 0.9345191194738147, 0.7431456520667159, 0.9152341819469156, 0.756664015301393, 0.9095934982523699]]} \ No newline at end of file diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0535b498557b89082849055550d002e1ade24b7e --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0912050a1ad9dbe66ae9cd19192f72af3835002662afcb6109b27cd1a30a67d +size 1442898 diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..494ca4f0ff754d032617a8011112fff3ea4d03ca --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d770d2c2408f04d7abb1e233b4ce8b6adc12ae2851e1e74d9b31b4d95ca67fe6 +size 1454280 diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..553374901c74f1b09bc75a5f34dc429895f59cdb --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b10b1d7fc921011b04f06948c7c2f9bf565d338709580df152bd2ff0e1090a +size 1470576 diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..03a09a314e96e160318c9b9b5a690ed78085d936 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4b0a7d2f5d2484c5d273f279f87160b09da9676192ce6ae529fcdc730947e2b +size 1472690 diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b503d533295d5b6831952f4c1e8280742890040 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2805344fc05b94c9bd94388782d346624b7dbcd9616124458c77d89ddd1f0846 +size 1483706 diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..03133bc44ab446a98bf077a046224d9a5bd071cf --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log @@ -0,0 +1,263 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736291183.772217]: resetting environment +[INFO] [1736291183.780407]: cleared current plan +[INFO] [1736291183.782022]: moving to home +[INFO] [1736291184.784597]: reached home +[INFO] [1736291184.785931]: reset action history +[INFO] [1736291187.546396]: environment reset +Starting episode 0 +[INFO] [1736291187.551168]: resetting environment +[INFO] [1736291187.556012]: cleared current plan +[INFO] [1736291187.556872]: moving to home +[INFO] [1736291188.559270]: reached home +[INFO] [1736291188.560913]: reset action history +[INFO] [1736291191.270425]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736291197.931794]: resetting environment +[INFO] [1736291197.938126]: cleared current plan +[INFO] [1736291197.938937]: moving to home +[INFO] [1736291198.941038]: reached home +[INFO] [1736291198.942493]: reset action history +[INFO] [1736291201.651379]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/episode_rosbags/episode_0_2025-01-07-18-06-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2207769.log +started roslaunch server http://158.130.50.37:41513/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2208073] +process[print_text-2]: started with pid [2208097] +process[rosbag_record-3]: started with pid [2208109] +started bagging! +For topic gripper_width: timestamp difference is 121706140 for nearest: 1736291220445480239 - target: 1736291220323774099 at idx 249 +For topic gripper_width: timestamp difference is -209764802 for nearest: 1736291222478786386 - target: 1736291222688551188 at idx 249 +[INFO] [1736291236.822527]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736291237.608022]: Stored episode 1. +[INFO] [1736291237.608281]: Saving video... +[INFO] [1736291237.790144]: Video saved! +Starting episode 1 +[INFO] [1736291237.805418]: resetting environment +[INFO] [1736291237.812944]: cleared current plan +[INFO] [1736291237.813195]: moving to home +[INFO] [1736291241.515950]: reached home +[INFO] [1736291241.516389]: reset action history +[INFO] [1736291244.225908]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736291248.492700]: resetting environment +[INFO] [1736291248.501483]: cleared current plan +[INFO] [1736291248.501974]: moving to home +[INFO] [1736291249.503999]: reached home +[INFO] [1736291249.504484]: reset action history +[INFO] [1736291252.213238]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/episode_rosbags/episode_1_2025-01-07-18-07-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2207769.log +started roslaunch server http://158.130.50.37:41089/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2208254] +process[print_text-5]: started with pid [2208255] +process[rosbag_record-6]: started with pid [2208256] +started bagging! +[INFO] [1736291287.631132]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736291288.422412]: Stored episode 2. +[INFO] [1736291288.423001]: Saving video... +[INFO] [1736291288.636881]: Video saved! +Starting episode 2 +[INFO] [1736291288.684993]: resetting environment +[INFO] [1736291288.700767]: cleared current plan +[INFO] [1736291288.701950]: moving to home +[INFO] [1736291292.406098]: reached home +[INFO] [1736291292.408531]: reset action history +[INFO] [1736291295.129516]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736291298.385302]: resetting environment +[INFO] [1736291298.402195]: cleared current plan +[INFO] [1736291298.403529]: moving to home +[INFO] [1736291299.405201]: reached home +[INFO] [1736291299.409085]: reset action history +[INFO] [1736291302.137094]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/episode_rosbags/episode_2_2025-01-07-18-08-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2207769.log +started roslaunch server http://158.130.50.37:41131/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2208397] +process[print_text-8]: started with pid [2208398] +process[rosbag_record-9]: started with pid [2208399] +started bagging! +For topic gripper_width: timestamp difference is 103550927 for nearest: 1736291321745502488 - target: 1736291321641951561 at idx 249 +[INFO] [1736291338.167541]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1736291338.958206]: Stored episode 3. +[INFO] [1736291338.968524]: Saving video... +[INFO] [1736291339.173775]: Video saved! +Starting episode 3 +[INFO] [1736291339.226855]: resetting environment +[INFO] [1736291339.245656]: cleared current plan +[INFO] [1736291339.246712]: moving to home +[INFO] [1736291342.949613]: reached home +[INFO] [1736291342.957001]: reset action history +[INFO] [1736291345.733297]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736291358.247303]: resetting environment +[INFO] [1736291358.270962]: cleared current plan +[INFO] [1736291358.272272]: moving to home +[INFO] [1736291359.277193]: reached home +[INFO] [1736291359.284943]: reset action history +[INFO] [1736291362.065091]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/episode_rosbags/episode_3_2025-01-07-18-09-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2207769.log +started roslaunch server http://158.130.50.37:40047/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2208538] +process[print_text-11]: started with pid [2208539] +process[rosbag_record-12]: started with pid [2208540] +started bagging! +[INFO] [1736291397.983493]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736291398.741684]: Stored episode 4. +[INFO] [1736291398.743707]: Saving video... +[INFO] [1736291398.953081]: Video saved! +Starting episode 4 +[INFO] [1736291399.005881]: resetting environment +[INFO] [1736291399.038834]: cleared current plan +[INFO] [1736291399.040783]: moving to home +[INFO] [1736291402.745203]: reached home +[INFO] [1736291402.750519]: reset action history +[INFO] [1736291405.531837]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736291409.602616]: resetting environment +[INFO] [1736291409.632649]: cleared current plan +[INFO] [1736291409.637757]: moving to home +[INFO] [1736291410.645966]: reached home +[INFO] [1736291410.652375]: reset action history +[INFO] [1736291413.399879]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/episode_rosbags/episode_4_2025-01-07-18-10-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2207769.log +started roslaunch server http://158.130.50.37:38689/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2208680] +process[print_text-14]: started with pid [2208681] +process[rosbag_record-15]: started with pid [2208682] +started bagging! +[INFO] [1736291449.391444]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1736291450.148694]: Stored episode 5. +[INFO] [1736291450.158841]: Saving video... +[INFO] [1736291450.375488]: Video saved! +[INFO] [1736291450.435871]: resetting environment +[INFO] [1736291450.461647]: cleared current plan +[INFO] [1736291450.464785]: moving to home +[INFO] [1736291454.174054]: reached home +[INFO] [1736291454.183205]: reset action history +[INFO] [1736291456.983034]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-metadata.json b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e259c0ab33b3fe4ce0c3e0ab6bb5d817bd9d1dee --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T23:06:19.739320", + "startedAt": "2025-01-07T23:06:19.347690", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3566.71675, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4502.251, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4509.34, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 802.8848648071289 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1f608464be57ba1d0c01fe2532dd4b5c2b630fb6 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "b0912050a1ad9dbe66ae9cd19192f72af3835002662afcb6109b27cd1a30a67d", "size": 1442898, "path": "media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4"}, "global_step": 665, "_timestamp": 1736291470.8592703, "_runtime": 291.50178837776184, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "d770d2c2408f04d7abb1e233b4ce8b6adc12ae2851e1e74d9b31b4d95ca67fe6", "size": 1454280, "path": "media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "a9b10b1d7fc921011b04f06948c7c2f9bf565d338709580df152bd2ff0e1090a", "size": 1470576, "path": "media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "b4b0a7d2f5d2484c5d273f279f87160b09da9676192ce6ae529fcdc730947e2b", "size": 1472690, "path": "media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "2805344fc05b94c9bd94388782d346624b7dbcd9616124458c77d89ddd1f0846", "size": 1483706, "path": "media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://qjruauqh2miamjosvw0ft67syytg01ifzff7w3qs68gtkbqjridkcw4657upv3uwcdqmbrxf2ngmv8q1c6j2s6dqh27oy5vmbvnhbdect4dqhcker2oew62y813ecsnu/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://pwclxx2wt6i94pyrm89fu7l0f2pt9z1hnttx9yxj9i7jheck2cnk407j3net49w7pk5h6jqn12cp1shwx5lmszczvmbngxne340ftaafkw1ml1igtb26ptyp8bzoj79c:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 38, "total_num_episodes": 45, "total_success_rate": 0.8444444444444444, "total_success_rate_lb_0.01": 0.6862424909696025, "total_success_rate_ub_0.01": 0.9418053179095713, "total_success_rate_lb_0.025": 0.7218401282582063, "total_success_rate_ub_0.025": 0.9345191194738147, "total_success_rate_lb_0.05": 0.7431456520667159, "total_success_rate_ub_0.05": 0.9152341819469156, "total_success_rate_lb_0.1": 0.756664015301393, "total_success_rate_ub_0.1": 0.9095934982523699, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "add3878bc9cb899afed4b5ee9cf5c7a3ae315d941efd1e3b1694ec4ff7976ec3", "size": 519, "artifact_path": "wandb-client-artifact://q9flj8gbxefxxjfjb6yp2sl3q7tfnhuhjqlc6bbbu01o1rf8d9futw01mptsxca5mvu8ikj0wrnxexos377sspikbtzdie6n8szy6dxd8rk05ln3oheozmj5hxbov4tn/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://p4zf1dwzu5k6tbulxm92a57v2m0x5lwf377cm2kdjwl6mhuxe5kf4gtt5bs641bteioj3liza89380xxgp6rftp1mhetmo3h7kjbot9upbzqmssbi4zlyzsyv1tm1wku:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 291}, "eval/total_success_rate": 0.8444444537162781, "eval/total_success_rate_lb_0.01": 0.6862424612045288, "eval/total_success_rate_ub_0.01": 0.9418053030967712, "eval/total_success_rate_lb_0.025": 0.7218401432037354, "eval/total_success_rate_ub_0.025": 0.9345191121101379, "eval/total_success_rate_lb_0.05": 0.7431456446647644, "eval/total_success_rate_ub_0.05": 0.9152342081069946, "eval/total_success_rate_lb_0.1": 0.7566640377044678, "eval/total_success_rate_ub_0.1": 0.9095935225486755} \ No newline at end of file diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug-internal.log b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..516a9a0d6e150a7ccac257410e473aa2b0cee700 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug-internal.log @@ -0,0 +1,800 @@ +2025-01-07 18:06:19,355 INFO StreamThr :2207962 [internal.py:wandb_internal():85] W&B internal server running at pid: 2207962, started at: 2025-01-07 18:06:19.355271 +2025-01-07 18:06:19,356 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:06:19,368 INFO WriterThread:2207962 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/run-hcdwvdao.wandb +2025-01-07 18:06:19,368 DEBUG SenderThread:2207962 [sender.py:send():379] send: header +2025-01-07 18:06:19,375 DEBUG SenderThread:2207962 [sender.py:send():379] send: run +2025-01-07 18:06:19,630 INFO SenderThread:2207962 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files +2025-01-07 18:06:19,631 INFO SenderThread:2207962 [sender.py:_start_run_threads():1188] run started: hcdwvdao with start time 1736291179.357482 +2025-01-07 18:06:19,646 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:06:19,647 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:06:19,720 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:06:19,725 DEBUG HandlerThread:2207962 [system_info.py:__init__():26] System info init +2025-01-07 18:06:19,725 DEBUG HandlerThread:2207962 [system_info.py:__init__():41] System info init done +2025-01-07 18:06:19,725 INFO HandlerThread:2207962 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:06:19,725 INFO SystemMonitor:2207962 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:06:19,725 INFO HandlerThread:2207962 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:06:19,726 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:06:19,727 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:06:19,727 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:06:19,728 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:06:19,728 INFO SystemMonitor:2207962 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:06:19,739 DEBUG HandlerThread:2207962 [system_info.py:probe():152] Probing system +2025-01-07 18:06:19,740 DEBUG HandlerThread:2207962 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:06:19,745 DEBUG HandlerThread:2207962 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:06:19,745 DEBUG HandlerThread:2207962 [system_info.py:probe():200] Probing system done +2025-01-07 18:06:19,745 DEBUG HandlerThread:2207962 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:06:19.739320', 'startedAt': '2025-01-07T23:06:19.347690', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3566.71675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4502.251, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4509.34, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 802.8848648071289}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:06:19,746 INFO HandlerThread:2207962 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:06:19,746 INFO HandlerThread:2207962 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:06:19,746 DEBUG HandlerThread:2207962 [system_info.py:_save_code():44] Saving code +2025-01-07 18:06:19,747 DEBUG HandlerThread:2207962 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:06:19,747 DEBUG HandlerThread:2207962 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:06:19,809 DEBUG HandlerThread:2207962 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:06:19,810 INFO HandlerThread:2207962 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:06:19,813 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:06:19,813 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:06:19,813 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:06:19,813 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 18:06:20,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:06:20,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:06:20,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:06:20,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:06:20,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:20,057 INFO HandlerThread:2207962 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/tb" + save: true +} +control { + relay_id: "55d9574faf0842428b21c40159837d41" +} +_info { + stream_id: "hcdwvdao" +} + +2025-01-07 18:06:20,057 DEBUG HandlerThread:2207962 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:06:20,068 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/eifv9h5e-code/FISH/eval_robot.py +2025-01-07 18:06:20,082 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/3cvz4p6z-wandb-metadata.json +2025-01-07 18:06:20,132 DEBUG SenderThread:2207962 [sender.py:send():379] send: telemetry +2025-01-07 18:06:20,132 DEBUG SenderThread:2207962 [sender.py:send():379] send: tbrecord +2025-01-07 18:06:20,132 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:06:20,133 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 with policy live +2025-01-07 18:06:20,142 INFO wandb-upload_2:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/3i0p729g-diff.patch +2025-01-07 18:06:20,321 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/h3heor1y-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:06:20,632 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:20,632 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt +2025-01-07 18:06:20,632 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH/eval_robot.py +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-metadata.json +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/diff.patch +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH +2025-01-07 18:06:20,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code +2025-01-07 18:06:21,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:22,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:22,631 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:23,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:23,772 DEBUG SenderThread:2207962 [sender.py:send():379] send: config +2025-01-07 18:06:24,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:24,785 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:25,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:26,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:26,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:27,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:28,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:28,633 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:29,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:30,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:30,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:30,634 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:31,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:32,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:32,634 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:33,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:34,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:35,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:06:35,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:06:35,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:35,115 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:36,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:37,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:38,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:39,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:40,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:40,637 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:41,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:41,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:42,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:42,637 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:43,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:44,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:44,638 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:06:45,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:46,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:46,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:47,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:48,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:49,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:50,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:06:50,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:06:50,101 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:51,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:51,169 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:51,640 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml +2025-01-07 18:06:52,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:53,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:54,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:55,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:56,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:57,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:06:57,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:58,060 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:06:59,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:00,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:01,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:02,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:02,497 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:02,642 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:03,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:04,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:04,643 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:05,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:05,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:05,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:06,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:07,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:08,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:08,153 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:09,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:10,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:11,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:12,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:13,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:14,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:14,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:15,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:16,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:17,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:17,802 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:07:17,802 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 with policy now +2025-01-07 18:07:17,803 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:07:17,806 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:07:17,806 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:07:17,807 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:07:18,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:18,067 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/xfkr53fc-media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:07:18,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media +2025-01-07 18:07:18,648 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos +2025-01-07 18:07:18,648 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:07:18,948 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/xmoxyftm-media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:07:19,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:19,728 DEBUG SystemMonitor:2207962 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:07:19,729 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:07:19,729 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:20,054 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:20,054 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:20,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:20,647 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:21,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:22,064 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:22,648 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:23,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:24,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:25,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:25,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:26,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:26,649 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:27,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:28,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:28,650 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:07:28,945 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/mwkiq0be-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:07:29,066 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:30,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:30,651 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:31,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:31,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:32,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:33,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:34,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:34,652 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:07:35,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:35,059 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:35,101 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:36,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:36,218 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:37,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:38,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:39,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:40,067 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:41,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:42,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:42,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:43,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:44,068 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:45,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:46,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:47,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:47,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:48,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:49,069 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:49,729 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:07:50,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:07:50,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:07:50,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:51,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:52,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:52,214 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:53,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:54,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:55,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:56,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:57,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:58,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:07:58,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:07:59,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:00,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:01,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:02,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:03,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:03,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:04,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:05,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:05,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:05,101 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:06,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:07,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:08,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:08,423 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:08,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:08,676 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:08:08,676 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 with policy now +2025-01-07 18:08:08,679 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:08:08,679 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:08:08,679 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:08:08,683 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:08:08,951 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/ejiokp1q-media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:08:09,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:09,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:08:09,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:08:09,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:08:09,984 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/qmp57b2j-media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:08:10,075 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:10,663 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:11,078 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:12,080 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:13,080 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:13,453 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:14,078 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:14,665 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:15,080 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:16,079 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:16,665 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:17,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:18,088 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:18,666 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:08:19,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:19,413 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:19,494 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:08:19,494 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:08:19,495 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:08:19,667 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:08:19,730 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:08:20,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:20,057 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:20,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:20,667 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:21,086 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:22,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:23,082 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:24,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:24,483 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:24,668 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:25,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:26,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:27,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:28,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:29,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:30,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:30,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:31,083 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:32,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:33,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:34,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:35,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:35,055 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:35,097 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:35,149 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:36,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:37,084 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:38,277 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:39,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:40,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:41,061 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:41,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:42,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:42,680 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:08:43,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:44,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:45,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:46,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:46,271 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:47,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:48,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:49,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:49,730 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:08:50,055 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:08:50,056 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:08:50,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:51,114 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:51,272 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:52,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:53,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:54,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:55,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:56,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:57,062 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:08:57,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:58,273 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:59,217 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:08:59,217 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 with policy now +2025-01-07 18:08:59,220 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:08:59,220 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:08:59,220 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:08:59,226 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:08:59,274 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:08:59,473 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/400suiaq-media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:08:59,687 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:08:59,688 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:08:59,688 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:08:59,964 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/7gfiypmj-media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:09:00,281 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:00,688 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:01,276 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:02,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:02,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:03,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:04,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:04,689 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:05,058 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:05,058 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:05,277 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:06,278 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:06,690 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:07,277 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:07,777 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:08,279 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:09,282 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:10,286 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:11,282 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:12,284 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:13,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:13,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:14,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:15,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:16,289 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:17,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:18,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:18,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:18,347 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:09:18,347 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:09:18,348 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:09:18,694 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:09:18,694 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:09:18,955 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/neeeybwj-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:09:19,321 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:19,731 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:09:20,057 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:20,057 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:20,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:20,694 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:21,295 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:22,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:23,280 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:23,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:24,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:24,696 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:09:25,287 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:26,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:27,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:28,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:29,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:29,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:30,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:31,288 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:32,289 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:33,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:34,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:34,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:35,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:35,057 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:35,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:36,290 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:37,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:38,291 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:39,118 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:39,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:40,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:41,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:42,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:43,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:44,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:45,070 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:45,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:46,292 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:47,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:48,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:49,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:49,731 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:09:50,056 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:09:50,056 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:09:50,211 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:50,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:51,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:52,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:53,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:54,293 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:55,294 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:56,071 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:09:56,294 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:57,295 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:58,295 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:58,999 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:09:58,999 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 with policy now +2025-01-07 18:09:59,002 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:09:59,002 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:09:59,003 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:09:59,009 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:09:59,262 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/5ymy2z1j-media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:09:59,296 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:09:59,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:09:59,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:09:59,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:09:59,989 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/6l876urn-media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:10:00,302 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:00,708 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:01,075 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:01,300 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:02,299 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:03,301 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:04,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:04,709 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:05,063 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:05,063 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:05,314 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:06,303 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:06,573 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:06,710 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:07,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:08,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:09,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:09,711 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:10:10,302 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:10,412 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:10:10,413 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:10:10,413 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:10:10,711 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:10:10,712 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:11,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:11,693 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:12,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:12,712 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:13,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:14,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:14,713 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:15,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:16,304 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:16,714 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:17,072 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:17,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:18,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:19,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:19,732 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:10:20,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:20,059 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:20,305 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:21,307 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:22,170 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:22,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:23,306 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:24,307 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:25,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:26,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:27,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:28,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:28,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:29,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:30,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:31,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:32,308 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:33,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:33,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:34,309 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:35,059 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:35,059 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:35,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:36,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:37,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:38,143 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:38,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:39,310 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:40,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:41,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:42,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:43,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:44,073 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:44,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:45,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:46,311 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:47,312 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:48,312 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:49,312 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:49,391 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:49,732 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:10:50,074 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:10:50,074 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:10:50,322 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:50,429 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:10:50,429 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 with policy now +2025-01-07 18:10:50,435 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:10:50,435 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:10:50,435 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:10:50,443 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:10:50,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval +2025-01-07 18:10:50,737 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/ioczzw4e-media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:10:51,009 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/dmn7kbli-media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:10:51,324 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:52,331 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:52,727 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:53,324 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:54,327 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:55,079 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:10:55,326 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:56,330 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:56,729 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:57,332 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:58,329 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:10:58,729 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:10:59,329 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:00,090 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:00,336 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:01,338 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:02,333 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:03,338 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:04,340 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:05,065 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:11:05,065 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:11:05,189 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:05,342 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:06,341 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:07,336 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:08,340 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:09,348 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:09,490 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:09,490 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:09,490 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:09,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:09,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:09,994 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/xj7zdd1l-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:10,313 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:11:10,313 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:11:10,357 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:10,553 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:11:10,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:10,733 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:10,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media +2025-01-07 18:11:10,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval +2025-01-07 18:11:10,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table +2025-01-07 18:11:10,820 INFO wandb-upload_1:2207962 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn0uzkz_a +2025-01-07 18:11:10,966 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:11:10,980 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:11:10,990 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/dc8zpfxj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:11,194 INFO SenderThread:2207962 [sender.py:send_request_log_artifact():1525] logged artifact run-hcdwvdao-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA2NjkyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc0MzgzNQ==', 'latestArtifact': None}} +2025-01-07 18:11:11,194 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:11:11,194 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 18:11:11,195 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:11,195 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:11,195 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,196 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,196 DEBUG SenderThread:2207962 [sender.py:send():379] send: telemetry +2025-01-07 18:11:11,196 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,200 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,200 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,200 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,200 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,200 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,201 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,201 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,201 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,202 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,202 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,202 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,202 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,203 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,203 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,203 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,203 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,204 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,204 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,204 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,204 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,205 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,205 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:11:11,338 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:11,466 INFO wandb-upload_2:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/gq1xcm8x-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:11,587 INFO wandb-upload_0:2207962 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa5g1zu2x +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:11,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval +2025-01-07 18:11:11,962 INFO SenderThread:2207962 [sender.py:send_request_log_artifact():1525] logged artifact run-hcdwvdao-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA2NjkzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc0MzgzNg==', 'latestArtifact': None}} +2025-01-07 18:11:11,962 DEBUG SenderThread:2207962 [sender.py:send():379] send: files +2025-01-07 18:11:11,962 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json with policy now +2025-01-07 18:11:11,962 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:11,963 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:11,963 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,963 DEBUG SenderThread:2207962 [sender.py:send():379] send: exit +2025-01-07 18:11:11,963 INFO SenderThread:2207962 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:11:11,963 INFO SenderThread:2207962 [sender.py:send_exit():588] handling runtime: 291 +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:send_exit():594] send defer +2025-01-07 18:11:11,964 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:11,964 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:11:11,964 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:11:11,964 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:11,964 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:11:11,964 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:11:11,964 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:11:11,965 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:11:11,965 DEBUG SystemMonitor:2207962 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:11:11,965 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:11:11,965 DEBUG SystemMonitor:2207962 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:11:12,017 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:11:12,018 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:11:12,018 INFO HandlerThread:2207962 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:11:12,018 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:12,018 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:11:12,018 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:11:12,018 DEBUG SenderThread:2207962 [sender.py:send():379] send: stats +2025-01-07 18:11:12,019 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:12,019 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:11:12,019 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:12,019 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:11:12,019 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:11:12,019 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:12,019 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:11:12,147 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/pta6s1go-media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:12,404 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/rmqd7loz-media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:12,734 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:12,735 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:14,735 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:18,492 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:18,492 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:18,492 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:11:18,493 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,493 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:11:18,493 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:11:18,494 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:18,494 DEBUG SenderThread:2207962 [sender.py:send():379] send: history +2025-01-07 18:11:18,494 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:11:18,494 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:18,495 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:11:18,496 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:18,496 DEBUG SenderThread:2207962 [sender.py:send():379] send: summary +2025-01-07 18:11:18,500 INFO SenderThread:2207962 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:11:18,500 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,500 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:11:18,500 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:11:18,500 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:18,501 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:11:18,501 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,501 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:11:18,504 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:11:18,633 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:11:18,633 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:18,633 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:11:18,634 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:18,634 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:11:18,737 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:18,737 INFO Thread-12 :2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml +2025-01-07 18:11:19,459 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:20,425 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:11:20,425 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:20,425 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:20,425 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:11:20,426 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:20,426 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:11:20,426 INFO SenderThread:2207962 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:11:20,428 INFO SenderThread:2207962 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:11:20,430 INFO SenderThread:2207962 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:11:20,445 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:11:20,445 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:20,445 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:11:20,445 DEBUG SenderThread:2207962 [sender.py:send():379] send: artifact +2025-01-07 18:11:20,459 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:20,670 INFO SenderThread:2207962 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:11:20,671 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:20,671 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:11:20,671 INFO SenderThread:2207962 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:11:20,738 INFO SenderThread:2207962 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:20,738 INFO SenderThread:2207962 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files +2025-01-07 18:11:20,738 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml config.yaml +2025-01-07 18:11:20,739 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json wandb-summary.json +2025-01-07 18:11:20,740 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:20,740 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt requirements.txt +2025-01-07 18:11:20,740 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:11:20,741 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log output.log +2025-01-07 18:11:20,741 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/diff.patch diff.patch +2025-01-07 18:11:20,742 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:11:20,742 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 media/videos/eval/1_eval_1_d770d2c2408f04d7abb1.mp4 +2025-01-07 18:11:20,742 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 media/videos/eval/0_eval_0_b0912050a1ad9dbe66ae.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 media/videos/eval/4_eval_4_2805344fc05b94c9bd94.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 media/videos/eval/3_eval_3_b4b0a7d2f5d2484c5d27.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 media/videos/eval/2_eval_2_a9b10b1d7fc921011b04.mp4 +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json media/table/eval/total_success_rate_ci_6_add3878bc9cb899afed4.table.json +2025-01-07 18:11:20,743 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:11:20,743 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:20,746 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:20,746 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:11:20,748 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:20,748 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:11:20,748 INFO SenderThread:2207962 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:11:20,933 INFO wandb-upload_1:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/config.yaml +2025-01-07 18:11:20,934 INFO wandb-upload_0:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/wandb-summary.json +2025-01-07 18:11:20,984 INFO wandb-upload_4:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/output.log +2025-01-07 18:11:20,984 INFO wandb-upload_3:2207962 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/files/requirements.txt +2025-01-07 18:11:21,035 INFO wandb-upload_2:2207962 [upload_job.py:push():130] Uploaded file /tmp/tmpo5oxjgeowandb/n4hex9g0-events.out.tfevents.1736291180.leonmkim-ROG-Strix-G15CS-G15CS.2207769.0 +2025-01-07 18:11:21,235 INFO Thread-11 (_thread_body):2207962 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:11:21,235 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,235 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:11:21,236 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,236 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:11:21,236 INFO SenderThread:2207962 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:11:21,237 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:11:21,237 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,237 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:11:21,237 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,237 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:11:21,237 INFO SenderThread:2207962 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:11:21,460 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:21,561 INFO SenderThread:2207962 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:11:21,561 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:11:21,561 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:21,561 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,561 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:11:21,561 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,562 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:11:21,562 INFO SenderThread:2207962 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:11:21,562 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:11:21,562 INFO HandlerThread:2207962 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:11:21,562 DEBUG SenderThread:2207962 [sender.py:send():379] send: final +2025-01-07 18:11:21,562 DEBUG SenderThread:2207962 [sender.py:send():379] send: footer +2025-01-07 18:11:21,563 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: defer +2025-01-07 18:11:21,563 INFO SenderThread:2207962 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:11:21,563 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:21,563 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:21,564 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:11:21,564 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:11:21,564 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:11:21,565 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:11:21,565 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:11:21,565 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:11:21,618 DEBUG SenderThread:2207962 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:11:21,618 INFO MainThread:2207962 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:11:21,619 INFO MainThread:2207962 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:11:21,619 INFO MainThread:2207962 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:11:21,619 DEBUG HandlerThread:2207962 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:11:21,619 INFO HandlerThread:2207962 [handler.py:finish():882] shutting down handler +2025-01-07 18:11:22,564 INFO WriterThread:2207962 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/run-hcdwvdao.wandb +2025-01-07 18:11:22,618 INFO SenderThread:2207962 [sender.py:finish():1615] shutting down sender +2025-01-07 18:11:22,618 INFO SenderThread:2207962 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:11:22,618 INFO SenderThread:2207962 [file_pusher.py:join():175] waiting for file pusher diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug.log b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..be7b335e02f5079935ac264b8f3bc4de68f2fdd8 --- /dev/null +++ b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Configure stats pid to 2207769 +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:06:19,349 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/settings +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug.log +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/logs/debug-internal.log +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao'} +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():619] starting backend +2025-01-07 18:06:19,350 INFO MainThread:2207769 [wandb_init.py:init():623] setting up manager +2025-01-07 18:06:19,354 INFO MainThread:2207769 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:06:19,356 INFO MainThread:2207769 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:06:19,367 INFO MainThread:2207769 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:06:19,373 INFO MainThread:2207769 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:06:19,646 INFO MainThread:2207769 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:06:19,714 INFO MainThread:2207769 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:06:19,714 INFO MainThread:2207769 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:06:20,054 INFO MainThread:2207769 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:06:20,054 INFO MainThread:2207769 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:06:20,055 INFO MainThread:2207769 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:06:20,055 INFO MainThread:2207769 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:06:20,056 INFO MainThread:2207769 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:06:20,057 INFO MainThread:2207769 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hcdwvdao/tb, True +2025-01-07 18:06:23,771 INFO MainThread:2207769 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:11:22,620 WARNING MsgRouterThr:2207769 [router.py:message_loop():77] message_loop has been closed diff --git a/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/run-hcdwvdao.wandb b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/run-hcdwvdao.wandb new file mode 100644 index 0000000000000000000000000000000000000000..100ba263cc0454ddd481bef65dacae42507dbc12 Binary files /dev/null and b/hcdwvdao/wandb/run-20250107_180619-hcdwvdao/run-hcdwvdao.wandb differ diff --git a/r2rih1d8/.hydra/config.yaml b/r2rih1d8/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/r2rih1d8/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/r2rih1d8/.hydra/hydra.yaml b/r2rih1d8/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4474529e8e278a349bf6d7c9f215b2fcc24c3b8a --- /dev/null +++ b/r2rih1d8/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/r2rih1d8/.hydra/overrides.yaml b/r2rih1d8/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/r2rih1d8/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/r2rih1d8/episode_rosbags/aligned_depth_to_color_K.npy b/r2rih1d8/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/r2rih1d8/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/r2rih1d8/episode_rosbags/cam_tf_world.npy b/r2rih1d8/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/r2rih1d8/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/r2rih1d8/episode_rosbags/color_K.npy b/r2rih1d8/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/r2rih1d8/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/r2rih1d8/episode_rosbags/depth_K.npy b/r2rih1d8/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/r2rih1d8/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/r2rih1d8/eval_robot.log b/r2rih1d8/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..260259b1247498e7d16975d4d333525f3c5f867b --- /dev/null +++ b/r2rih1d8/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 16:46:08,481][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 16:46:08,484][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/r2rih1d8/eval_video/0_eval.mp4 b/r2rih1d8/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..84cb78d0b76115ae407456ea541e5f163285eed0 --- /dev/null +++ b/r2rih1d8/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9bbe64239f681549779c44dab6f905fdfeab48e6dd61ed818e6c6574c2a1d0c +size 1475109 diff --git a/r2rih1d8/eval_video/1_eval.mp4 b/r2rih1d8/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70ed2abe7d0f9cf3b5e7ebe1a42395d8d0a84060 --- /dev/null +++ b/r2rih1d8/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19311f99aa7344d97b7ca724db4524cb0c0a5eb7796712885eb9053fe7cc006e +size 1468473 diff --git a/r2rih1d8/eval_video/2_eval.mp4 b/r2rih1d8/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb6c5c05d40ba3b5d7a3587cd31a4a02c3fcde33 --- /dev/null +++ b/r2rih1d8/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7399e211c13789e38c74ca54e15bc9d9f4aced54cb0b69d22109fa574e394025 +size 1469039 diff --git a/r2rih1d8/eval_video/3_eval.mp4 b/r2rih1d8/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..017809f430fdbe2d12ace255f76605119f0d2ce7 --- /dev/null +++ b/r2rih1d8/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842f285a0cc010f660efa8405bdca7f20af6e2a762d388e0004d9b90c084ea76 +size 1474193 diff --git a/r2rih1d8/eval_video/4_eval.mp4 b/r2rih1d8/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53d197341cb09a9e2ccd40c94ddf756cebbc445e --- /dev/null +++ b/r2rih1d8/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2a8587f620a40e056b3ff83efc94c1b459cf483bcc4cd340de988c92d2c59c +size 1441789 diff --git a/r2rih1d8/tb/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 b/r2rih1d8/tb/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 new file mode 100644 index 0000000000000000000000000000000000000000..7cbfa7abcdb4adeda99ee2246f3a3b1bd493b455 --- /dev/null +++ b/r2rih1d8/tb/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a25ca065c0188865d92fb52e971359b43995eb0a12a1405f748f2e96f21f3e7a +size 6539 diff --git a/r2rih1d8/wandb/debug-internal.log b/r2rih1d8/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9e672b5f99e3a24693bcdd0a21f6f401748d1c3d --- /dev/null +++ b/r2rih1d8/wandb/debug-internal.log @@ -0,0 +1,811 @@ +2025-01-07 16:46:14,154 INFO StreamThr :2188977 [internal.py:wandb_internal():85] W&B internal server running at pid: 2188977, started at: 2025-01-07 16:46:14.154192 +2025-01-07 16:46:14,155 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:46:14,173 INFO WriterThread:2188977 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/run-r2rih1d8.wandb +2025-01-07 16:46:14,173 DEBUG SenderThread:2188977 [sender.py:send():379] send: header +2025-01-07 16:46:14,181 DEBUG SenderThread:2188977 [sender.py:send():379] send: run +2025-01-07 16:46:14,523 INFO SenderThread:2188977 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files +2025-01-07 16:46:14,523 INFO SenderThread:2188977 [sender.py:_start_run_threads():1188] run started: r2rih1d8 with start time 1736286374.155248 +2025-01-07 16:46:14,545 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:46:14,545 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:46:14,628 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:46:14,635 DEBUG HandlerThread:2188977 [system_info.py:__init__():26] System info init +2025-01-07 16:46:14,635 DEBUG HandlerThread:2188977 [system_info.py:__init__():41] System info init done +2025-01-07 16:46:14,635 INFO HandlerThread:2188977 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:46:14,635 INFO SystemMonitor:2188977 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:46:14,635 INFO HandlerThread:2188977 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:46:14,636 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:46:14,636 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:46:14,637 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:46:14,638 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:46:14,639 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:46:14,648 DEBUG HandlerThread:2188977 [system_info.py:probe():152] Probing system +2025-01-07 16:46:14,649 DEBUG HandlerThread:2188977 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_info.py:probe():200] Probing system done +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:46:14.648457', 'startedAt': '2025-01-07T21:46:14.140991', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.1028750000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.952, 'min': 800.0, 'max': 4700.0}, {'current': 4498.886, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 699.4591331481934}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:46:14,657 INFO HandlerThread:2188977 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:46:14,657 INFO HandlerThread:2188977 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_info.py:_save_code():44] Saving code +2025-01-07 16:46:14,660 DEBUG HandlerThread:2188977 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:46:14,660 DEBUG HandlerThread:2188977 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:46:14,726 DEBUG HandlerThread:2188977 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:46:14,727 INFO HandlerThread:2188977 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:46:14,730 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:46:14,730 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:46:14,731 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:46:14,731 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 16:46:14,960 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:46:14,960 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:46:14,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:14,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:14,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:46:14,970 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/rgzlx3ls-wandb-metadata.json +2025-01-07 16:46:14,993 INFO wandb-upload_2:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/skv2iqfy-diff.patch +2025-01-07 16:46:14,996 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/fmg4256f-code/FISH/eval_robot.py +2025-01-07 16:46:15,005 INFO HandlerThread:2188977 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/tb" + save: true +} +control { + relay_id: "02b63335b8f44a64a3e34fef6ead4c73" +} +_info { + stream_id: "r2rih1d8" +} + +2025-01-07 16:46:15,005 DEBUG HandlerThread:2188977 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:46:15,086 DEBUG SenderThread:2188977 [sender.py:send():379] send: telemetry +2025-01-07 16:46:15,086 DEBUG SenderThread:2188977 [sender.py:send():379] send: tbrecord +2025-01-07 16:46:15,087 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:46:15,087 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 with policy live +2025-01-07 16:46:15,256 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/ettnqzfz-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:46:15,524 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-metadata.json +2025-01-07 16:46:15,524 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH/eval_robot.py +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/diff.patch +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH +2025-01-07 16:46:15,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:16,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:17,524 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:17,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:18,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:19,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:19,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:20,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:21,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:22,091 DEBUG SenderThread:2188977 [sender.py:send():379] send: config +2025-01-07 16:46:22,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:23,526 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:23,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:24,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:24,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:25,526 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:25,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:26,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:27,527 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:27,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:28,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:29,960 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:29,960 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:46:29,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:30,041 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:30,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:31,528 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:31,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:32,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:33,528 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:33,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:34,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:35,156 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:35,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:36,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:37,530 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:37,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:38,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:39,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:40,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:40,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:41,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:42,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:43,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:44,960 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:44,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:46:45,001 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:45,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:46,021 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:46,532 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml +2025-01-07 16:46:46,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:47,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:48,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:49,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:50,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:51,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:51,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:52,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:53,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:54,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:55,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:56,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:57,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:57,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:58,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:59,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:59,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:00,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:00,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:01,967 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:02,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:03,105 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:03,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:04,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:05,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:06,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:07,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:08,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:08,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:09,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:10,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:11,485 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:47:11,485 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 with policy now +2025-01-07 16:47:11,485 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:47:11,486 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:47:11,486 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:47:11,486 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:47:11,540 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:11,540 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:47:11,709 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/f0v5myng-media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:47:11,952 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/pk3272l0-media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:47:11,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:12,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:13,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:13,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:14,497 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:14,639 DEBUG SystemMonitor:2188977 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:47:14,640 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:47:14,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:14,963 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:14,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:15,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:16,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:17,542 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:17,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:18,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:19,543 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:19,653 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:19,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:20,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:21,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:22,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:23,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:24,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:24,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:25,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:26,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:27,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:28,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:29,546 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:29,546 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:47:29,785 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/5545pmis-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:47:29,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:29,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:29,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:30,045 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:30,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:31,547 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:31,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:32,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:33,548 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:33,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:34,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:35,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:35,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:36,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:37,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:38,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:39,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:40,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:41,093 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:41,550 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:41,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:42,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:43,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:44,640 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:47:44,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:44,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:45,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:45,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:46,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:47,037 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:47,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:48,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:49,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:50,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:51,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:52,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:52,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:53,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:54,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:55,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:56,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:57,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:57,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:58,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:59,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:59,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:00,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:00,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:01,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:02,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:03,083 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:03,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:04,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:05,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:06,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:07,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:08,401 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:08,469 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:48:08,469 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 with policy now +2025-01-07 16:48:08,470 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:48:08,470 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:48:08,470 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:48:08,481 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:48:08,559 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:48:08,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:48:08,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:48:08,701 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/i2uq9ai0-media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:48:08,965 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/rn6qy6v1-media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:48:08,979 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:09,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:09,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:10,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:10,984 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:11,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:12,984 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:13,561 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:13,841 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:13,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:14,640 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:48:14,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:14,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:14,984 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:15,562 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:15,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:16,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:17,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:18,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:19,093 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:19,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:20,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:21,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:22,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:23,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:24,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:24,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:25,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:26,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:27,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:28,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:29,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:29,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:30,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:30,089 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:30,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:31,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:32,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:33,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:34,442 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:48:34,442 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:48:34,443 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:48:34,568 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:48:34,568 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:48:34,818 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/i8hkbrpf-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:48:35,001 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:35,568 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:35,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:36,001 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:36,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:37,569 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:37,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:38,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:39,570 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:39,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:40,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:41,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:41,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:42,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:43,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:44,641 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:48:44,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:44,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:45,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:45,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:46,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:47,035 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:47,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:48,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:49,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:50,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:51,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:52,967 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:52,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:53,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:54,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:55,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:56,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:57,576 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:57,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:58,389 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:58,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:59,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:59,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:00,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:00,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:01,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:02,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:03,578 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:03,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:04,093 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:04,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:05,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:06,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:07,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:08,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:09,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:09,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:10,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:11,581 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:11,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:12,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:13,582 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:13,941 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:49:13,942 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 with policy now +2025-01-07 16:49:13,945 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:49:13,946 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:49:13,946 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:49:13,946 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:49:13,994 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:14,157 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/o8iu0kg4-media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:49:14,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:49:14,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:49:14,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:49:14,641 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:49:14,849 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/imfgrjhb-media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:49:14,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:14,971 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:15,013 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:15,127 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:15,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:15,996 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:17,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:18,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:18,998 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:19,584 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:19,999 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:20,145 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:21,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:21,585 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:22,004 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:23,000 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:24,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:25,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:25,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:26,010 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:27,003 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:28,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:29,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:29,292 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:49:29,292 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:49:29,292 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:49:29,588 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:29,588 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:49:29,588 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:49:29,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:29,968 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:30,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:31,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:31,089 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:31,589 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:32,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:33,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:33,590 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:34,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:35,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:35,591 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:36,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:36,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:37,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:38,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:39,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:40,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:41,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:41,593 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:42,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:42,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:43,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:44,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:44,642 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:49:44,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:44,966 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:45,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:45,595 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:46,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:47,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:48,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:48,114 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:49,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:50,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:51,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:52,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:53,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:53,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:54,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:55,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:56,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:57,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:58,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:59,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:59,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:59,966 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:00,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:00,091 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:01,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:02,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:03,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:04,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:05,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:05,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:06,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:07,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:08,011 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:08,912 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:50:08,912 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 with policy now +2025-01-07 16:50:08,916 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:50:08,917 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:50:08,917 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:50:08,922 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:50:09,011 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:09,136 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/gkv1aj9f-media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:50:09,881 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/ijq1k2h2-media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:50:10,018 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:10,604 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:10,976 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:11,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:11,604 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:12,021 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:13,023 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:13,605 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:14,022 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:14,642 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:50:14,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:14,969 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:15,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:16,023 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:16,381 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:17,022 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:17,606 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:18,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:19,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:20,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:20,607 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:50:20,847 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/qpeigcze-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:50:21,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:21,359 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:50:21,359 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:50:21,360 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:50:21,433 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:21,607 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:21,607 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:50:22,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:23,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:23,608 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:24,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:25,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:25,609 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:26,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:26,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:27,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:28,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:29,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:29,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:29,966 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:30,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:31,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:32,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:32,095 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:33,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:34,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:35,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:36,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:37,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:37,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:38,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:39,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:40,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:41,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:42,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:42,981 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:43,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:44,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:44,643 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:50:44,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:44,967 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:45,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:46,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:47,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:48,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:48,092 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:49,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:50,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:51,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:52,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:53,028 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:53,976 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:54,028 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:55,030 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:56,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:57,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:58,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:59,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:59,054 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:59,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:59,968 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:51:00,033 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:00,189 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:51:00,189 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 with policy now +2025-01-07 16:51:00,190 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:51:00,190 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:00,191 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:00,194 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:00,411 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/bpebeoai-media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:00,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:00,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:00,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:51:00,863 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/qmoz3y1o-media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:01,037 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:01,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:02,059 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:03,052 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:04,055 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:04,789 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:05,060 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:05,623 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:06,063 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:07,055 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:07,623 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:08,059 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:08,624 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:09,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:51:09,002 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:51:09,054 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:09,155 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:51:09,581 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:51:09,590 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:51:09,624 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval +2025-01-07 16:51:09,629 INFO wandb-upload_0:2188977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkro896e6 +2025-01-07 16:51:09,808 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/398jm2zk-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:09,824 INFO wandb-upload_2:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/twt4x7p6-media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:10,038 INFO SenderThread:2188977 [sender.py:send_request_log_artifact():1525] logged artifact run-r2rih1d8-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTgyODY1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY3NDM3NQ==', 'latestArtifact': None}} +2025-01-07 16:51:10,038 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:51:10,038 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-07 16:51:10,038 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:10,039 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,039 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:10,039 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,039 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:10,040 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,040 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,040 DEBUG SenderThread:2188977 [sender.py:send():379] send: telemetry +2025-01-07 16:51:10,040 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,040 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,041 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,041 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,043 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,043 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,043 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,043 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,044 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,044 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,044 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,044 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,044 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,045 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,045 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,045 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,045 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,046 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,046 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,046 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,046 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,047 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,047 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:51:10,054 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:10,303 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/aiqyygp9-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:10,413 INFO wandb-upload_2:2188977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpawg3mwr8 +2025-01-07 16:51:10,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:10,800 INFO SenderThread:2188977 [sender.py:send_request_log_artifact():1525] logged artifact run-r2rih1d8-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTgyODY5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY3NDM3OQ==', 'latestArtifact': None}} +2025-01-07 16:51:10,800 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:51:10,801 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json with policy now +2025-01-07 16:51:10,801 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:10,801 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,801 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,802 DEBUG SenderThread:2188977 [sender.py:send():379] send: exit +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:send_exit():588] handling runtime: 296 +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:send_exit():594] send defer +2025-01-07 16:51:10,803 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,803 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 16:51:10,804 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,804 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 16:51:10,804 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 1 +2025-01-07 16:51:10,804 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,804 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 16:51:10,805 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,805 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 16:51:10,805 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 2 +2025-01-07 16:51:10,805 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,805 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 16:51:10,805 INFO HandlerThread:2188977 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 16:51:10,805 DEBUG SystemMonitor:2188977 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 16:51:10,805 DEBUG SystemMonitor:2188977 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 16:51:10,805 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 16:51:10,806 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 16:51:10,861 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 16:51:10,862 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 16:51:10,862 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined network monitor +2025-01-07 16:51:10,862 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,862 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 16:51:10,862 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 3 +2025-01-07 16:51:10,863 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:51:10,863 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,863 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 16:51:10,863 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,863 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 16:51:10,863 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 4 +2025-01-07 16:51:10,863 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,863 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 16:51:11,042 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/rylk8xxf-media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:11,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:11,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:17,420 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:17,420 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:17,420 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 16:51:17,421 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,421 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 16:51:17,421 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 5 +2025-01-07 16:51:17,421 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:17,421 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:17,422 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 16:51:17,422 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:17,422 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:17,422 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:17,423 DEBUG SenderThread:2188977 [sender.py:send():379] send: summary +2025-01-07 16:51:17,423 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:17,423 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,423 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 16:51:17,423 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 6 +2025-01-07 16:51:17,423 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:17,423 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 16:51:17,424 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,424 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 16:51:17,427 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:17,568 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 7 +2025-01-07 16:51:17,568 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:17,568 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 16:51:17,568 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,568 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 16:51:17,629 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:17,629 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml +2025-01-07 16:51:17,659 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:20,672 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 8 +2025-01-07 16:51:20,672 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:20,672 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:20,672 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 16:51:20,672 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:20,672 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 16:51:20,672 INFO SenderThread:2188977 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 16:51:20,675 INFO SenderThread:2188977 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 16:51:20,677 INFO SenderThread:2188977 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 16:51:20,686 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 9 +2025-01-07 16:51:20,687 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:20,687 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 16:51:20,687 DEBUG SenderThread:2188977 [sender.py:send():379] send: artifact +2025-01-07 16:51:21,111 INFO wandb-upload_1:2188977 [upload_job.py:push():85] Skipped uploading /tmp/tmpwq1i9mlc/wandb-job.json +2025-01-07 16:51:21,111 INFO wandb-upload_0:2188977 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2p1l5b62 +2025-01-07 16:51:21,226 INFO wandb-upload_2:2188977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn5_eqeh9 +2025-01-07 16:51:21,629 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:21,660 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:21,782 INFO SenderThread:2188977 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'versionIndex': 432}}} +2025-01-07 16:51:21,783 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:21,783 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 16:51:21,783 INFO SenderThread:2188977 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 16:51:22,630 INFO SenderThread:2188977 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files +2025-01-07 16:51:22,630 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml config.yaml +2025-01-07 16:51:22,630 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json wandb-summary.json +2025-01-07 16:51:22,632 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt requirements.txt +2025-01-07 16:51:22,634 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-metadata.json wandb-metadata.json +2025-01-07 16:51:22,634 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:22,636 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log output.log +2025-01-07 16:51:22,636 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/diff.patch diff.patch +2025-01-07 16:51:22,636 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:51:22,645 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:22,645 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:22,645 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 10 +2025-01-07 16:51:22,646 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:22,646 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:22,646 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 16:51:22,646 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:22,646 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 16:51:22,646 INFO SenderThread:2188977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:51:22,661 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:22,661 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:22,825 INFO wandb-upload_2:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:22,828 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt +2025-01-07 16:51:22,833 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml +2025-01-07 16:51:22,900 INFO wandb-upload_4:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:22,953 INFO wandb-upload_3:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/qjwlgepw-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:23,153 INFO Thread-11 (_thread_body):2188977 [sender.py:transition_state():614] send defer: 11 +2025-01-07 16:51:23,153 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,153 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 16:51:23,154 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,154 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 16:51:23,154 INFO SenderThread:2188977 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 16:51:23,155 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 12 +2025-01-07 16:51:23,155 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,155 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 16:51:23,155 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,155 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 16:51:23,155 INFO SenderThread:2188977 [file_stream.py:finish():601] file stream finish called +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [file_stream.py:finish():605] file stream finish is done +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 13 +2025-01-07 16:51:23,374 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,374 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 16:51:23,374 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 14 +2025-01-07 16:51:23,374 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,374 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 16:51:23,375 DEBUG SenderThread:2188977 [sender.py:send():379] send: final +2025-01-07 16:51:23,375 DEBUG SenderThread:2188977 [sender.py:send():379] send: footer +2025-01-07 16:51:23,375 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,375 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 16:51:23,375 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:23,376 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 16:51:23,376 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:23,376 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 16:51:23,376 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:23,377 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: server_info +2025-01-07 16:51:23,378 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 16:51:23,379 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:23,437 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:23,437 INFO MainThread:2188977 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 16:51:23,438 INFO MainThread:2188977 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 16:51:23,438 INFO MainThread:2188977 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 16:51:23,438 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 16:51:23,438 INFO HandlerThread:2188977 [handler.py:finish():882] shutting down handler +2025-01-07 16:51:24,376 INFO WriterThread:2188977 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/run-r2rih1d8.wandb +2025-01-07 16:51:24,437 INFO SenderThread:2188977 [sender.py:finish():1615] shutting down sender +2025-01-07 16:51:24,437 INFO SenderThread:2188977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:51:24,437 INFO SenderThread:2188977 [file_pusher.py:join():175] waiting for file pusher diff --git a/r2rih1d8/wandb/debug.log b/r2rih1d8/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..da9e44ab1bfe29fb26cda28e52de259b2882603a --- /dev/null +++ b/r2rih1d8/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Configure stats pid to 2188797 +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/settings +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug.log +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug-internal.log +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8'} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():619] starting backend +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():623] setting up manager +2025-01-07 16:46:14,153 INFO MainThread:2188797 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:46:14,154 INFO MainThread:2188797 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:46:14,173 INFO MainThread:2188797 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:46:14,179 INFO MainThread:2188797 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:46:14,544 INFO MainThread:2188797 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:46:14,617 INFO MainThread:2188797 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:46:14,617 INFO MainThread:2188797 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:46:14,960 INFO MainThread:2188797 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:46:14,961 INFO MainThread:2188797 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:46:14,961 INFO MainThread:2188797 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:46:14,961 INFO MainThread:2188797 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:46:14,962 INFO MainThread:2188797 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:46:14,962 INFO MainThread:2188797 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/tb, True +2025-01-07 16:46:22,091 INFO MainThread:2188797 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 16:51:24,439 WARNING MsgRouterThr:2188797 [router.py:message_loop():77] message_loop has been closed diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH/eval_robot.py b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d674f9f433cd42c0e076cde7de3ba21a9157be6 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1003_0' +wandb_notes: + desc: null + value: 1003_0_req_1364_0restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736286374 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/diff.patch b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..4697b4ebb6e44a4ea64b46b459fc2cf4999dc3f0 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/diff.patch @@ -0,0 +1,16 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..1643b06 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,9 +138,9 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +-# wandb_run_id: '1003_0' ++wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cc7e5e91447669fe989ab08b6e5b18934b0ea7ca --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.04400911936343024, 0.8621948956718848, 0.05324718365067565, 0.803534984527916, 0.11712844677476308, 0.7487166672447589, 0.23282015328982517, 0.6637700199753331]]} \ No newline at end of file diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c40fa26b38c3c53bdc66a22778125ef015689021 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[32, 35, 0.9142857142857143, 0.7502509057021318, 0.984712358467196, 0.7922011464249812, 0.9815873763492143, 0.814573354956178, 0.9679677653121805, 0.8247396647280667, 0.9666311106642631]]} \ No newline at end of file diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..84cb78d0b76115ae407456ea541e5f163285eed0 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9bbe64239f681549779c44dab6f905fdfeab48e6dd61ed818e6c6574c2a1d0c +size 1475109 diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70ed2abe7d0f9cf3b5e7ebe1a42395d8d0a84060 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19311f99aa7344d97b7ca724db4524cb0c0a5eb7796712885eb9053fe7cc006e +size 1468473 diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb6c5c05d40ba3b5d7a3587cd31a4a02c3fcde33 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7399e211c13789e38c74ca54e15bc9d9f4aced54cb0b69d22109fa574e394025 +size 1469039 diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..017809f430fdbe2d12ace255f76605119f0d2ce7 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842f285a0cc010f660efa8405bdca7f20af6e2a762d388e0004d9b90c084ea76 +size 1474193 diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53d197341cb09a9e2ccd40c94ddf756cebbc445e --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2a8587f620a40e056b3ff83efc94c1b459cf483bcc4cd340de988c92d2c59c +size 1441789 diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5724cf414e59ff57bbe15778b7d78b67aa7d6883 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736286382.092060]: resetting environment +[INFO] [1736286382.096865]: cleared current plan +[INFO] [1736286382.098252]: moving to home +[INFO] [1736286383.100868]: reached home +[INFO] [1736286383.102359]: reset action history +[INFO] [1736286385.815969]: environment reset +Starting episode 0 +[INFO] [1736286385.820768]: resetting environment +[INFO] [1736286385.824809]: cleared current plan +[INFO] [1736286385.825645]: moving to home +[INFO] [1736286386.827874]: reached home +[INFO] [1736286386.829176]: reset action history +[INFO] [1736286389.539015]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736286391.435050]: resetting environment +[INFO] [1736286391.440637]: cleared current plan +[INFO] [1736286391.442289]: moving to home +[INFO] [1736286392.444530]: reached home +[INFO] [1736286392.446395]: reset action history +[INFO] [1736286395.155895]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/episode_rosbags/episode_0_2025-01-07-16-46-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2188797.log +started roslaunch server http://158.130.50.37:35563/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2189081] +process[print_text-2]: started with pid [2189105] +process[rosbag_record-3]: started with pid [2189106] +started bagging! +[INFO] [1736286430.454921]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736286431.242079]: Stored episode 1. +[INFO] [1736286431.244911]: Saving video... +[INFO] [1736286431.442830]: Video saved! +Starting episode 1 +[INFO] [1736286431.486344]: resetting environment +[INFO] [1736286431.496171]: cleared current plan +[INFO] [1736286431.496568]: moving to home +[INFO] [1736286434.900348]: reached home +[INFO] [1736286434.900872]: reset action history +[INFO] [1736286437.610029]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286448.606493]: resetting environment +[INFO] [1736286448.610219]: cleared current plan +[INFO] [1736286448.610619]: moving to home +[INFO] [1736286449.612489]: reached home +[INFO] [1736286449.612916]: reset action history +[INFO] [1736286452.318900]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/episode_rosbags/episode_1_2025-01-07-16-47-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2188797.log +started roslaunch server http://158.130.50.37:36425/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2189230] +process[print_text-5]: started with pid [2189231] +process[rosbag_record-6]: started with pid [2189255] +started bagging! +For topic gripper_width: timestamp difference is 112984011 for nearest: 1736286460045529196 - target: 1736286459932545185 at idx 107 +[INFO] [1736286487.144939]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736286487.858981]: Stored episode 2. +[INFO] [1736286487.861898]: Saving video... +[INFO] [1736286488.398501]: Video saved! +Starting episode 2 +[INFO] [1736286488.472623]: resetting environment +[INFO] [1736286488.489419]: cleared current plan +[INFO] [1736286488.489640]: moving to home +[WARN] [1736286488.487484]: Plan exhausted +[INFO] [1736286491.794090]: reached home +[INFO] [1736286491.795922]: reset action history +[INFO] [1736286494.575891]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286513.947731]: resetting environment +[INFO] [1736286513.956821]: cleared current plan +[INFO] [1736286513.957068]: moving to home +[INFO] [1736286514.960072]: reached home +[INFO] [1736286514.960531]: reset action history +[INFO] [1736286517.669473]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/episode_rosbags/episode_2_2025-01-07-16-48-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2188797.log +started roslaunch server http://158.130.50.37:37027/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2189402] +process[print_text-8]: started with pid [2189403] +process[rosbag_record-9]: started with pid [2189404] +started bagging! +For topic gripper_width: timestamp difference is 103653698 for nearest: 1736286535345505981 - target: 1736286535241852283 at idx 242 +For topic gripper_width: timestamp difference is 108275164 for nearest: 1736286541045526732 - target: 1736286540937251568 at idx 249 +For topic gripper_width: timestamp difference is -184129661 for nearest: 1736286550278884942 - target: 1736286550463014603 at idx 249 +For topic gripper_width: timestamp difference is 117538766 for nearest: 1736286552645524816 - target: 1736286552527986050 at idx 249 +[INFO] [1736286552.862197]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736286553.652728]: Stored episode 3. +[INFO] [1736286553.654794]: Saving video... +[INFO] [1736286553.904920]: Video saved! +Starting episode 3 +[INFO] [1736286553.948211]: resetting environment +[INFO] [1736286553.961408]: cleared current plan +[INFO] [1736286553.961659]: moving to home +[INFO] [1736286557.365105]: reached home +[INFO] [1736286557.368128]: reset action history +[INFO] [1736286560.094368]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286568.783677]: resetting environment +[INFO] [1736286568.805666]: cleared current plan +[INFO] [1736286568.809747]: moving to home +[INFO] [1736286569.817326]: reached home +[INFO] [1736286569.821548]: reset action history +[INFO] [1736286572.541468]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/episode_rosbags/episode_3_2025-01-07-16-49-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2188797.log +started roslaunch server http://158.130.50.37:34923/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2189541] +process[print_text-11]: started with pid [2189542] +process[rosbag_record-12]: started with pid [2189543] +started bagging! +For topic gripper_width: timestamp difference is 106094693 for nearest: 1736286579745553588 - target: 1736286579639458895 at idx 102 +For topic gripper_width: timestamp difference is 109900499 for nearest: 1736286584445523525 - target: 1736286584335623026 at idx 163 +[INFO] [1736286607.605636]: Storing episode... +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[INFO] [1736286608.401097]: Stored episode 4. +[INFO] [1736286608.410098]: Saving video... +[WARN] [1736286608.923175]: Plan exhausted +[INFO] [1736286608.876425]: Video saved! +Starting episode 4 +[INFO] [1736286608.918917]: resetting environment +[INFO] [1736286608.938343]: cleared current plan +[INFO] [1736286608.938577]: moving to home +[INFO] [1736286612.550363]: reached home +[INFO] [1736286612.551264]: reset action history +[INFO] [1736286615.338375]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286620.396328]: resetting environment +[INFO] [1736286620.428035]: cleared current plan +[INFO] [1736286620.430353]: moving to home +[INFO] [1736286621.432829]: reached home +[INFO] [1736286621.435230]: reset action history +[INFO] [1736286624.207675]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/episode_rosbags/episode_4_2025-01-07-16-50-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2188797.log +started roslaunch server http://158.130.50.37:41573/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2189660] +process[print_text-14]: started with pid [2189661] +process[rosbag_record-15]: started with pid [2189685] +started bagging! +[INFO] [1736286658.962804]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736286659.795580]: Stored episode 5. +[INFO] [1736286659.803233]: Saving video... +[INFO] [1736286660.132654]: Video saved! +[INFO] [1736286660.191754]: resetting environment +[INFO] [1736286660.235101]: cleared current plan +[INFO] [1736286660.235335]: moving to home +[INFO] [1736286663.744833]: reached home +[INFO] [1736286663.746260]: reset action history +[INFO] [1736286666.531610]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-metadata.json b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..044170588dfa2db69948b18f3e8423f35d5d6064 --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T21:46:14.648457", + "startedAt": "2025-01-07T21:46:14.140991", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4124.1028750000005, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.996, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.952, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.886, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 699.4591331481934 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5eafb5a05619bb4e50bf412cbea4cf29d587182e --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "a9bbe64239f681549779c44dab6f905fdfeab48e6dd61ed818e6c6574c2a1d0c", "size": 1475109, "path": "media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4"}, "global_step": 665, "_timestamp": 1736286669.4720044, "_runtime": 295.3167564868927, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "19311f99aa7344d97b7ca724db4524cb0c0a5eb7796712885eb9053fe7cc006e", "size": 1468473, "path": "media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4"}, "eval/num_success": 2.0, "episode": 35.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.044009119272232056, "eval/success_rate_ub_0.01": 0.8621948957443237, "eval/success_rate_lb_0.025": 0.05324718356132507, "eval/success_rate_ub_0.025": 0.803534984588623, "eval/success_rate_lb_0.05": 0.11712844669818878, "eval/success_rate_ub_0.05": 0.7487166523933411, "eval/success_rate_lb_0.1": 0.23282015323638916, "eval/success_rate_ub_0.1": 0.6637700200080872, "eval/2_eval": {"_type": "video-file", "sha256": "7399e211c13789e38c74ca54e15bc9d9f4aced54cb0b69d22109fa574e394025", "size": 1469039, "path": "media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "842f285a0cc010f660efa8405bdca7f20af6e2a762d388e0004d9b90c084ea76", "size": 1474193, "path": "media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "7d2a8587f620a40e056b3ff83efc94c1b459cf483bcc4cd340de988c92d2c59c", "size": 1441789, "path": "media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "85c54431b4eb7c8145a72a6437cbffccb84a9cf38051f18b07dd0a638d459f25", "size": 401, "artifact_path": "wandb-client-artifact://71bs6v7fq69kra17wasq7g724ab4tj9oabqs3jkvvx40yawm32ifjzc1l4d7a0281vpetg3k95lt700bvibmo58mmae7f8sgy8p99qcpf1l3g4n1lannfj16ytk612mc/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://r0jc7xtdkc4vdghlo858efaqlu9zi7l895ntiur08akuttmlbuh5d0d4nbw08107aix8gl62ii799a4d56papgwxwkpetlsedpgkmt1gk7lw716iq3tnai35rdby6t27:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 32, "total_num_episodes": 35, "total_success_rate": 0.9142857142857143, "total_success_rate_lb_0.01": 0.7502509057021318, "total_success_rate_ub_0.01": 0.984712358467196, "total_success_rate_lb_0.025": 0.7922011464249812, "total_success_rate_ub_0.025": 0.9815873763492143, "total_success_rate_lb_0.05": 0.814573354956178, "total_success_rate_ub_0.05": 0.9679677653121805, "total_success_rate_lb_0.1": 0.8247396647280667, "total_success_rate_ub_0.1": 0.9666311106642631, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "395e0921721327c8b7f6e8bb1445c2fa17f07ba5dab1279d74e774bb9ae20212", "size": 518, "artifact_path": "wandb-client-artifact://0n4mh4ktjl8hntorsmt8iw1klszq9yrv5320icgf6zg0cqo5l4v5du5v64as4v8eki20qmj30nyk92h95knj73nuvcby7iven9v2ojotwafwa5vnp0zbnuklya1xh895/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://r4a0qjxi7w7rdpuprb4wabom5y02ex5hvokqxw3irgiu3ia1dz07qyszfrerclgqs522xoou0erjd226zx43esaidp0z4yl3tcndt2j2hiy18371b73xl8mv8lt7oz3a:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 296}, "eval/total_success_rate": 0.9142857193946838, "eval/total_success_rate_lb_0.01": 0.7502508759498596, "eval/total_success_rate_ub_0.01": 0.9847123622894287, "eval/total_success_rate_lb_0.025": 0.7922011613845825, "eval/total_success_rate_ub_0.025": 0.9815873503684998, "eval/total_success_rate_lb_0.05": 0.814573347568512, "eval/total_success_rate_ub_0.05": 0.9679677486419678, "eval/total_success_rate_lb_0.1": 0.8247396349906921, "eval/total_success_rate_ub_0.1": 0.9666311144828796} \ No newline at end of file diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug-internal.log b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9e672b5f99e3a24693bcdd0a21f6f401748d1c3d --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug-internal.log @@ -0,0 +1,811 @@ +2025-01-07 16:46:14,154 INFO StreamThr :2188977 [internal.py:wandb_internal():85] W&B internal server running at pid: 2188977, started at: 2025-01-07 16:46:14.154192 +2025-01-07 16:46:14,155 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:46:14,173 INFO WriterThread:2188977 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/run-r2rih1d8.wandb +2025-01-07 16:46:14,173 DEBUG SenderThread:2188977 [sender.py:send():379] send: header +2025-01-07 16:46:14,181 DEBUG SenderThread:2188977 [sender.py:send():379] send: run +2025-01-07 16:46:14,523 INFO SenderThread:2188977 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files +2025-01-07 16:46:14,523 INFO SenderThread:2188977 [sender.py:_start_run_threads():1188] run started: r2rih1d8 with start time 1736286374.155248 +2025-01-07 16:46:14,545 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:46:14,545 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:46:14,628 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:46:14,635 DEBUG HandlerThread:2188977 [system_info.py:__init__():26] System info init +2025-01-07 16:46:14,635 DEBUG HandlerThread:2188977 [system_info.py:__init__():41] System info init done +2025-01-07 16:46:14,635 INFO HandlerThread:2188977 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:46:14,635 INFO SystemMonitor:2188977 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:46:14,635 INFO HandlerThread:2188977 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:46:14,636 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:46:14,636 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:46:14,637 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:46:14,638 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:46:14,639 INFO SystemMonitor:2188977 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:46:14,648 DEBUG HandlerThread:2188977 [system_info.py:probe():152] Probing system +2025-01-07 16:46:14,649 DEBUG HandlerThread:2188977 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_info.py:probe():200] Probing system done +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:46:14.648457', 'startedAt': '2025-01-07T21:46:14.140991', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.1028750000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.952, 'min': 800.0, 'max': 4700.0}, {'current': 4498.886, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 699.4591331481934}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:46:14,657 INFO HandlerThread:2188977 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:46:14,657 INFO HandlerThread:2188977 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:46:14,657 DEBUG HandlerThread:2188977 [system_info.py:_save_code():44] Saving code +2025-01-07 16:46:14,660 DEBUG HandlerThread:2188977 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:46:14,660 DEBUG HandlerThread:2188977 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:46:14,726 DEBUG HandlerThread:2188977 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:46:14,727 INFO HandlerThread:2188977 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:46:14,730 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:46:14,730 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:46:14,731 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:46:14,731 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 16:46:14,960 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:46:14,960 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:46:14,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:14,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:14,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:46:14,970 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/rgzlx3ls-wandb-metadata.json +2025-01-07 16:46:14,993 INFO wandb-upload_2:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/skv2iqfy-diff.patch +2025-01-07 16:46:14,996 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/fmg4256f-code/FISH/eval_robot.py +2025-01-07 16:46:15,005 INFO HandlerThread:2188977 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/tb" + save: true +} +control { + relay_id: "02b63335b8f44a64a3e34fef6ead4c73" +} +_info { + stream_id: "r2rih1d8" +} + +2025-01-07 16:46:15,005 DEBUG HandlerThread:2188977 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:46:15,086 DEBUG SenderThread:2188977 [sender.py:send():379] send: telemetry +2025-01-07 16:46:15,086 DEBUG SenderThread:2188977 [sender.py:send():379] send: tbrecord +2025-01-07 16:46:15,087 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:46:15,087 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 with policy live +2025-01-07 16:46:15,256 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/ettnqzfz-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:46:15,524 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-metadata.json +2025-01-07 16:46:15,524 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH/eval_robot.py +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/diff.patch +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code +2025-01-07 16:46:15,525 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH +2025-01-07 16:46:15,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:16,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:17,524 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:17,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:18,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:19,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:19,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:20,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:21,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:22,091 DEBUG SenderThread:2188977 [sender.py:send():379] send: config +2025-01-07 16:46:22,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:23,526 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:23,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:24,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:24,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:25,526 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:25,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:26,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:27,527 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:27,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:28,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:29,960 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:29,960 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:46:29,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:30,041 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:30,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:31,528 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:31,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:32,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:33,528 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:33,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:34,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:35,156 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:35,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:36,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:37,530 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:46:37,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:38,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:39,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:40,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:40,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:41,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:42,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:43,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:44,960 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:44,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:46:45,001 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:45,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:46,021 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:46,532 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml +2025-01-07 16:46:46,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:47,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:48,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:49,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:50,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:51,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:51,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:52,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:53,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:54,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:55,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:56,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:57,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:46:57,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:58,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:46:59,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:46:59,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:00,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:00,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:01,967 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:02,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:03,105 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:03,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:04,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:05,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:06,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:07,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:08,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:08,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:09,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:10,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:11,485 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:47:11,485 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 with policy now +2025-01-07 16:47:11,485 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:47:11,486 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:47:11,486 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:47:11,486 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:47:11,540 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:11,540 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos +2025-01-07 16:47:11,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:47:11,709 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/f0v5myng-media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:47:11,952 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/pk3272l0-media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:47:11,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:12,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:13,541 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:13,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:14,497 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:14,639 DEBUG SystemMonitor:2188977 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:47:14,640 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:47:14,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:14,963 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:14,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:15,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:16,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:17,542 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:17,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:18,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:19,543 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:19,653 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:19,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:20,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:21,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:22,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:23,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:24,963 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:24,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:25,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:26,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:27,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:28,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:29,546 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:29,546 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:47:29,785 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/5545pmis-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:47:29,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:29,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:29,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:30,045 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:30,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:31,547 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:31,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:32,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:33,548 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:33,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:34,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:35,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:35,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:36,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:37,972 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:38,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:39,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:40,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:41,093 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:41,550 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:47:41,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:42,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:43,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:44,640 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:47:44,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:44,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:47:45,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:45,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:46,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:47,037 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:47,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:48,974 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:49,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:50,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:51,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:52,964 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:52,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:53,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:54,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:55,975 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:56,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:57,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:47:57,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:58,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:47:59,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:47:59,961 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:00,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:00,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:01,977 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:02,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:03,083 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:03,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:04,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:05,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:06,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:07,978 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:08,401 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:08,469 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:48:08,469 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 with policy now +2025-01-07 16:48:08,470 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:48:08,470 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:48:08,470 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:48:08,481 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:48:08,559 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:48:08,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:48:08,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:48:08,701 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/i2uq9ai0-media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:48:08,965 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/rn6qy6v1-media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:48:08,979 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:09,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:09,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:10,560 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:10,984 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:11,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:12,984 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:13,561 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:13,841 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:13,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:14,640 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:48:14,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:14,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:14,984 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:15,562 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:15,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:16,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:17,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:18,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:19,093 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:19,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:20,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:21,985 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:22,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:23,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:24,965 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:24,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:25,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:26,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:27,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:28,986 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:29,961 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:29,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:30,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:30,089 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:30,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:31,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:32,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:33,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:34,442 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:48:34,442 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:48:34,443 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:48:34,568 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:48:34,568 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:48:34,818 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/i8hkbrpf-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:48:35,001 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:35,568 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:35,987 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:36,001 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:36,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:37,569 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:37,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:38,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:39,570 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:39,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:40,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:41,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:41,988 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:42,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:43,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:44,641 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:48:44,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:44,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:48:45,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:45,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:46,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:47,035 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:47,989 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:48,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:49,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:50,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:51,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:52,967 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:52,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:53,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:54,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:55,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:56,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:57,576 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:48:57,990 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:58,389 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:48:58,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:48:59,962 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:48:59,962 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:00,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:00,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:01,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:02,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:03,578 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:03,991 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:04,093 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:04,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:05,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:06,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:07,992 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:08,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:09,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:09,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:10,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:11,581 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:11,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:12,993 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:13,582 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:13,941 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:49:13,942 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 with policy now +2025-01-07 16:49:13,945 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:49:13,946 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:49:13,946 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:49:13,946 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:49:13,994 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:14,157 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/o8iu0kg4-media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:49:14,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:49:14,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:49:14,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:49:14,641 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:49:14,849 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/imfgrjhb-media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:49:14,970 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:14,971 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:15,013 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:15,127 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:15,583 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:15,996 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:17,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:18,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:18,998 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:19,584 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:19,999 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:20,145 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:21,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:21,585 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:22,004 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:23,000 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:24,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:25,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:25,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:26,010 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:27,003 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:28,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:29,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:29,292 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:49:29,292 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:49:29,292 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:49:29,588 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:29,588 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:49:29,588 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:49:29,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:29,968 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:30,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:31,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:31,089 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:31,589 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:32,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:33,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:33,590 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:34,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:35,005 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:35,591 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:36,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:36,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:37,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:38,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:39,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:40,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:41,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:41,593 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:42,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:42,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:43,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:44,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:44,642 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:49:44,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:44,966 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:49:45,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:45,595 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:49:46,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:47,006 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:48,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:48,114 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:49,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:50,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:51,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:52,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:53,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:53,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:49:54,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:55,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:56,007 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:57,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:58,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:59,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:49:59,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:49:59,966 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:00,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:00,091 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:01,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:02,008 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:03,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:04,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:05,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:05,971 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:06,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:07,009 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:08,011 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:08,912 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:50:08,912 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 with policy now +2025-01-07 16:50:08,916 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:50:08,917 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:50:08,917 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:50:08,922 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:50:09,011 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:09,136 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/gkv1aj9f-media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:50:09,603 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:50:09,881 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/ijq1k2h2-media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:50:10,018 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:10,604 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:10,976 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:11,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:11,604 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:12,021 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:13,023 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:13,605 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:14,022 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:14,642 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:50:14,969 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:14,969 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:15,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:16,023 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:16,381 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:17,022 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:17,606 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:18,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:19,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:20,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:20,607 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:50:20,847 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/qpeigcze-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:50:21,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:21,359 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:50:21,359 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:50:21,360 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:50:21,433 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:21,607 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:21,607 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:50:22,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:23,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:23,608 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:24,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:25,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:25,609 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:50:26,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:26,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:27,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:28,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:29,024 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:29,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:29,966 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:30,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:31,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:32,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:32,095 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:33,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:34,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:35,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:36,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:37,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:37,973 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:38,025 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:39,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:40,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:41,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:42,026 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:42,981 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:43,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:44,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:44,643 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:50:44,966 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:44,967 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:50:45,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:46,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:47,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:48,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:48,092 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:49,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:50,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:51,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:52,027 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:53,028 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:53,976 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:54,028 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:55,030 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:56,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:57,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:58,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:59,029 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:50:59,054 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:50:59,968 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:50:59,968 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:51:00,033 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:00,189 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:51:00,189 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 with policy now +2025-01-07 16:51:00,190 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:51:00,190 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:00,191 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:00,194 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:00,411 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/bpebeoai-media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:00,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:00,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:00,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval +2025-01-07 16:51:00,863 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/qmoz3y1o-media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:01,037 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:01,621 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:02,059 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:03,052 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:04,055 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:04,789 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:05,060 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:05,623 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:06,063 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:07,055 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:07,623 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:08,059 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:08,624 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:09,002 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:51:09,002 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:51:09,054 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:09,155 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:51:09,581 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:51:09,590 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:51:09,624 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table +2025-01-07 16:51:09,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval +2025-01-07 16:51:09,629 INFO wandb-upload_0:2188977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkro896e6 +2025-01-07 16:51:09,808 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/398jm2zk-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:09,824 INFO wandb-upload_2:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/twt4x7p6-media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:10,038 INFO SenderThread:2188977 [sender.py:send_request_log_artifact():1525] logged artifact run-r2rih1d8-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTgyODY1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY3NDM3NQ==', 'latestArtifact': None}} +2025-01-07 16:51:10,038 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:51:10,038 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-07 16:51:10,038 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:10,039 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,039 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:10,039 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,039 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:10,040 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,040 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,040 DEBUG SenderThread:2188977 [sender.py:send():379] send: telemetry +2025-01-07 16:51:10,040 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,040 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,041 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,041 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,043 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,043 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,043 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,043 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,044 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,044 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,044 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,044 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,044 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,045 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,045 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,045 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,045 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,046 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,046 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,046 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,046 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,047 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,047 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:51:10,054 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:10,303 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/aiqyygp9-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:10,413 INFO wandb-upload_2:2188977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpawg3mwr8 +2025-01-07 16:51:10,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:10,800 INFO SenderThread:2188977 [sender.py:send_request_log_artifact():1525] logged artifact run-r2rih1d8-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTgyODY5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY3NDM3OQ==', 'latestArtifact': None}} +2025-01-07 16:51:10,800 DEBUG SenderThread:2188977 [sender.py:send():379] send: files +2025-01-07 16:51:10,801 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json with policy now +2025-01-07 16:51:10,801 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:10,801 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:10,801 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,802 DEBUG SenderThread:2188977 [sender.py:send():379] send: exit +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:send_exit():588] handling runtime: 296 +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:10,802 INFO SenderThread:2188977 [sender.py:send_exit():594] send defer +2025-01-07 16:51:10,803 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,803 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 16:51:10,804 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,804 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 16:51:10,804 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 1 +2025-01-07 16:51:10,804 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,804 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 16:51:10,805 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,805 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 16:51:10,805 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 2 +2025-01-07 16:51:10,805 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,805 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 16:51:10,805 INFO HandlerThread:2188977 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 16:51:10,805 DEBUG SystemMonitor:2188977 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 16:51:10,805 DEBUG SystemMonitor:2188977 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 16:51:10,805 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 16:51:10,806 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 16:51:10,861 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 16:51:10,862 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 16:51:10,862 INFO HandlerThread:2188977 [interfaces.py:finish():200] Joined network monitor +2025-01-07 16:51:10,862 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,862 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 16:51:10,862 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 3 +2025-01-07 16:51:10,863 DEBUG SenderThread:2188977 [sender.py:send():379] send: stats +2025-01-07 16:51:10,863 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,863 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 16:51:10,863 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:10,863 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 16:51:10,863 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 4 +2025-01-07 16:51:10,863 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:10,863 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 16:51:11,042 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/rylk8xxf-media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:11,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:11,625 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:17,420 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:17,420 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:17,420 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 16:51:17,421 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,421 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 16:51:17,421 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 5 +2025-01-07 16:51:17,421 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:17,421 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:17,422 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 16:51:17,422 DEBUG SenderThread:2188977 [sender.py:send():379] send: history +2025-01-07 16:51:17,422 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:51:17,422 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:17,423 DEBUG SenderThread:2188977 [sender.py:send():379] send: summary +2025-01-07 16:51:17,423 INFO SenderThread:2188977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:51:17,423 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,423 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 16:51:17,423 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 6 +2025-01-07 16:51:17,423 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:17,423 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 16:51:17,424 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,424 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 16:51:17,427 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:51:17,568 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 7 +2025-01-07 16:51:17,568 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:17,568 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 16:51:17,568 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:17,568 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 16:51:17,629 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:17,629 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml +2025-01-07 16:51:17,659 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:20,672 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 8 +2025-01-07 16:51:20,672 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:20,672 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:20,672 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 16:51:20,672 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:20,672 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 16:51:20,672 INFO SenderThread:2188977 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 16:51:20,675 INFO SenderThread:2188977 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 16:51:20,677 INFO SenderThread:2188977 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 16:51:20,686 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 9 +2025-01-07 16:51:20,687 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:20,687 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 16:51:20,687 DEBUG SenderThread:2188977 [sender.py:send():379] send: artifact +2025-01-07 16:51:21,111 INFO wandb-upload_1:2188977 [upload_job.py:push():85] Skipped uploading /tmp/tmpwq1i9mlc/wandb-job.json +2025-01-07 16:51:21,111 INFO wandb-upload_0:2188977 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2p1l5b62 +2025-01-07 16:51:21,226 INFO wandb-upload_2:2188977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn5_eqeh9 +2025-01-07 16:51:21,629 INFO Thread-12 :2188977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:21,660 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:21,782 INFO SenderThread:2188977 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'versionIndex': 432}}} +2025-01-07 16:51:21,783 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:21,783 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 16:51:21,783 INFO SenderThread:2188977 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 16:51:22,630 INFO SenderThread:2188977 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files +2025-01-07 16:51:22,630 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml config.yaml +2025-01-07 16:51:22,630 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json wandb-summary.json +2025-01-07 16:51:22,632 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt requirements.txt +2025-01-07 16:51:22,634 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-metadata.json wandb-metadata.json +2025-01-07 16:51:22,634 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:22,636 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log output.log +2025-01-07 16:51:22,636 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/diff.patch diff.patch +2025-01-07 16:51:22,636 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 media/videos/eval/2_eval_2_7399e211c13789e38c74.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 media/videos/eval/1_eval_1_19311f99aa7344d97b7c.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 media/videos/eval/3_eval_3_842f285a0cc010f660ef.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 media/videos/eval/4_eval_4_7d2a8587f620a40e056b.mp4 +2025-01-07 16:51:22,641 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 media/videos/eval/0_eval_0_a9bbe64239f681549779.mp4 +2025-01-07 16:51:22,645 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json media/table/eval/total_success_rate_ci_6_395e0921721327c8b7f6.table.json +2025-01-07 16:51:22,645 INFO SenderThread:2188977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 16:51:22,645 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 10 +2025-01-07 16:51:22,646 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:22,646 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:22,646 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 16:51:22,646 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:22,646 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 16:51:22,646 INFO SenderThread:2188977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:51:22,661 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:22,661 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:22,825 INFO wandb-upload_2:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/wandb-summary.json +2025-01-07 16:51:22,828 INFO wandb-upload_0:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/requirements.txt +2025-01-07 16:51:22,833 INFO wandb-upload_1:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/config.yaml +2025-01-07 16:51:22,900 INFO wandb-upload_4:2188977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/files/output.log +2025-01-07 16:51:22,953 INFO wandb-upload_3:2188977 [upload_job.py:push():130] Uploaded file /tmp/tmpkz8l6llywandb/qjwlgepw-events.out.tfevents.1736286374.leonmkim-ROG-Strix-G15CS-G15CS.2188797.0 +2025-01-07 16:51:23,153 INFO Thread-11 (_thread_body):2188977 [sender.py:transition_state():614] send defer: 11 +2025-01-07 16:51:23,153 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,153 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 16:51:23,154 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,154 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 16:51:23,154 INFO SenderThread:2188977 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 16:51:23,155 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 12 +2025-01-07 16:51:23,155 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,155 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 16:51:23,155 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,155 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 16:51:23,155 INFO SenderThread:2188977 [file_stream.py:finish():601] file stream finish called +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [file_stream.py:finish():605] file stream finish is done +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 13 +2025-01-07 16:51:23,374 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,374 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 16:51:23,374 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 16:51:23,374 INFO SenderThread:2188977 [sender.py:transition_state():614] send defer: 14 +2025-01-07 16:51:23,374 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:51:23,374 INFO HandlerThread:2188977 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 16:51:23,375 DEBUG SenderThread:2188977 [sender.py:send():379] send: final +2025-01-07 16:51:23,375 DEBUG SenderThread:2188977 [sender.py:send():379] send: footer +2025-01-07 16:51:23,375 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: defer +2025-01-07 16:51:23,375 INFO SenderThread:2188977 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 16:51:23,375 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:23,376 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 16:51:23,376 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:51:23,376 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 16:51:23,376 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:23,377 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: server_info +2025-01-07 16:51:23,378 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 16:51:23,379 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:51:23,437 DEBUG SenderThread:2188977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:51:23,437 INFO MainThread:2188977 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 16:51:23,438 INFO MainThread:2188977 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 16:51:23,438 INFO MainThread:2188977 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 16:51:23,438 DEBUG HandlerThread:2188977 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 16:51:23,438 INFO HandlerThread:2188977 [handler.py:finish():882] shutting down handler +2025-01-07 16:51:24,376 INFO WriterThread:2188977 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/run-r2rih1d8.wandb +2025-01-07 16:51:24,437 INFO SenderThread:2188977 [sender.py:finish():1615] shutting down sender +2025-01-07 16:51:24,437 INFO SenderThread:2188977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:51:24,437 INFO SenderThread:2188977 [file_pusher.py:join():175] waiting for file pusher diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug.log b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..da9e44ab1bfe29fb26cda28e52de259b2882603a --- /dev/null +++ b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Configure stats pid to 2188797 +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:46:14,147 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/settings +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug.log +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/logs/debug-internal.log +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8'} +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():619] starting backend +2025-01-07 16:46:14,148 INFO MainThread:2188797 [wandb_init.py:init():623] setting up manager +2025-01-07 16:46:14,153 INFO MainThread:2188797 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:46:14,154 INFO MainThread:2188797 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:46:14,173 INFO MainThread:2188797 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:46:14,179 INFO MainThread:2188797 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:46:14,544 INFO MainThread:2188797 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:46:14,617 INFO MainThread:2188797 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:46:14,617 INFO MainThread:2188797 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:46:14,960 INFO MainThread:2188797 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:46:14,961 INFO MainThread:2188797 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:46:14,961 INFO MainThread:2188797 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:46:14,961 INFO MainThread:2188797 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:46:14,962 INFO MainThread:2188797 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:46:14,962 INFO MainThread:2188797 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/r2rih1d8/tb, True +2025-01-07 16:46:22,091 INFO MainThread:2188797 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 16:51:24,439 WARNING MsgRouterThr:2188797 [router.py:message_loop():77] message_loop has been closed diff --git a/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/run-r2rih1d8.wandb b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/run-r2rih1d8.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0304fd714b36e14178574beef3b4f7d225bd5267 Binary files /dev/null and b/r2rih1d8/wandb/run-20250107_164614-r2rih1d8/run-r2rih1d8.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..94a804e6e5001de1d1dc835738565ec301349a8c --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9fdc43cb4cefe93b44627760cfd5f0abc9c19807d16602734a1cd8933f5352f +size 910445958 diff --git a/snapshot_11653.pt b/snapshot_11653.pt new file mode 100644 index 0000000000000000000000000000000000000000..fadfe90231dc571c64aa000c9e9cafedaff26ecf --- /dev/null +++ b/snapshot_11653.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf217d8078da3bd12b261a522cd88f293d75a56a4db7838689af5c2114f633be +size 910445958 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..08f8c38f5df63c5a0518d243c05a0ca08044aaf1 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:284dd5da040d498e2ca0e01acbd9561bc75a4b166993ac2cad0457859cf84deb +size 910445958 diff --git a/snapshot_12999.pt b/snapshot_12999.pt new file mode 100644 index 0000000000000000000000000000000000000000..caa31858c1e49a4a6275c2c20a7b4e2f36f2dc61 --- /dev/null +++ b/snapshot_12999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37f588d87faeb7ff592e1f37ab1f57409b9d21d05033eaded5d53e92db95b95d +size 910445958 diff --git a/snapshot_13249.pt b/snapshot_13249.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb0fcb6955a1cdb14d67bd9a818a57fa23d32b27 --- /dev/null +++ b/snapshot_13249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0549d3aa8c80ea367c59027b075d0256272b3ab174ce553b2061fa0f5366c939 +size 910445958 diff --git a/snapshot_14462.pt b/snapshot_14462.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9eb65345b4186a8ad9af3dfb4c45ed224ad19bd --- /dev/null +++ b/snapshot_14462.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fb934a6e26185c11136409e41543ee773ca8861538134ea33356eac6db3de3d +size 910445958 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..9541ed1b579c2ddc267af703a870513ace63c930 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10c3fbddaea71912faab89a87064d960fb0f04268845ad3f5a74599fdd2bd3b2 +size 910445958 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..484ab7cd88e347f395d08b1e6d50f5ca84ac809b --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79ed2379cadbad10cd2aa2c1a273d2e7c94f155c7484925204379fdbb92909c7 +size 910445958 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..604e74b8ee08a4782757262113d6c0acfa0f5da8 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:864feab6cf128a045c924e8d7988543c58cfc1f9b5b7cc3e307faa30b5c0e2d5 +size 910445958 diff --git a/snapshot_6998.pt b/snapshot_6998.pt new file mode 100644 index 0000000000000000000000000000000000000000..657310ef54a4d3bf157a816b03f209c363ae120d --- /dev/null +++ b/snapshot_6998.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de55cda9462229c9281c5d6804f0a88c8840efc901f96a02838a450f60429c5 +size 910445958 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..762c6e7950f3db503849ef4eee331867c90bec01 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a763b1f9cc763e854611dde57980c6dafdcc6c868aaa94026745de6d076b2d75 +size 910445958 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..807481df735a2579d94b048a80911e5c2173f9c4 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4513e3db0ccec06a8b8ecbc719f0be8da4e9589c802f402248491aeafadecec8 +size 910445958 diff --git a/snapshot_9499.pt b/snapshot_9499.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7df0078109c26029c4eb6fbfe72326f85209a70 --- /dev/null +++ b/snapshot_9499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f18d747231a756e62099dd7f6a9136f7acb68745173cfc082ea761b1002a1295 +size 910445958 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..9541ed1b579c2ddc267af703a870513ace63c930 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10c3fbddaea71912faab89a87064d960fb0f04268845ad3f5a74599fdd2bd3b2 +size 910445958