diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..1496f5b7b49b8e4d2257950f842a62e78c24ba05 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,38 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nyyc47dd/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 filter=lfs diff=lfs merge=lfs -text +w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 filter=lfs diff=lfs merge=lfs -text +w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 filter=lfs diff=lfs merge=lfs -text +w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 filter=lfs diff=lfs merge=lfs -text +w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/2bf37081/.hydra/config.yaml b/2bf37081/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/2bf37081/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/2bf37081/.hydra/hydra.yaml b/2bf37081/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b90fa2d9b6dc1a298988559091c292cba00be58 --- /dev/null +++ b/2bf37081/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/2bf37081 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/2bf37081/.hydra/overrides.yaml b/2bf37081/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/2bf37081/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/2bf37081/eval_policy.log b/2bf37081/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..40aba7c2461f58637e929651f23a1ad39563ac43 --- /dev/null +++ b/2bf37081/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 17:45:38,655][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:45:38,659][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 17:45:42,185][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/5qmtbx9d/.hydra/config.yaml b/5qmtbx9d/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/5qmtbx9d/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/5qmtbx9d/.hydra/hydra.yaml b/5qmtbx9d/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ee2c90ba25194f5c4b80882b097a9c3dbc67e95 --- /dev/null +++ b/5qmtbx9d/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/5qmtbx9d/.hydra/overrides.yaml b/5qmtbx9d/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/5qmtbx9d/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/5qmtbx9d/episode_rosbags/aligned_depth_to_color_K.npy b/5qmtbx9d/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5qmtbx9d/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5qmtbx9d/episode_rosbags/cam_tf_world.npy b/5qmtbx9d/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/5qmtbx9d/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/5qmtbx9d/episode_rosbags/color_K.npy b/5qmtbx9d/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5qmtbx9d/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5qmtbx9d/episode_rosbags/depth_K.npy b/5qmtbx9d/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/5qmtbx9d/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/5qmtbx9d/eval_robot.log b/5qmtbx9d/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..707b8e89a0ddbb3167f879a57d332118634c39cf --- /dev/null +++ b/5qmtbx9d/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:44:45,275][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:44:45,278][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/5qmtbx9d/eval_video/0_eval.mp4 b/5qmtbx9d/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e2363a734237f10dab2acb709fd7ef1a2e3ba6a --- /dev/null +++ b/5qmtbx9d/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:310b504403e8beda16b23ccfe0eebdf0935145269ae9a1ba2bd2c15f13727a16 +size 1146404 diff --git a/5qmtbx9d/eval_video/1_eval.mp4 b/5qmtbx9d/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c90ff384afc72c15b16f04b05fbbfd84bf1dfc97 --- /dev/null +++ b/5qmtbx9d/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c060e05047508277ede7722f7f3a0797eb47f21deae6e589348459a43ecc7931 +size 1209168 diff --git a/5qmtbx9d/eval_video/2_eval.mp4 b/5qmtbx9d/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d1ce764cb547a5adafdf823256b00bc6911c7f6 --- /dev/null +++ b/5qmtbx9d/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7a9172564184809716447311f702d7b29617121735a2e139cc8d1d58b2b922d +size 1181538 diff --git a/5qmtbx9d/eval_video/3_eval.mp4 b/5qmtbx9d/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60efe2058d01191c9920ffc6d68c7c5ee96dfd79 --- /dev/null +++ b/5qmtbx9d/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f11c37a5819a4ad2728beb9c63a371103173857454e736393d23a2544f058d4c +size 1194205 diff --git a/5qmtbx9d/eval_video/4_eval.mp4 b/5qmtbx9d/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfd5c897fb4c082fa713684b227a72d247fdf27f --- /dev/null +++ b/5qmtbx9d/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34d4110b1ed92fa8405fd4ab00f15027d790141e6db4d43d1a07c96206eb1c63 +size 1209669 diff --git a/5qmtbx9d/tb/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 b/5qmtbx9d/tb/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 new file mode 100644 index 0000000000000000000000000000000000000000..819100cd1c09ddea4aa65cb1314ce0848390dda4 --- /dev/null +++ b/5qmtbx9d/tb/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7151e27c725eb0d705f1824dd4bba3b73e68ec6e2bae71db410a2df8f7a1b6b +size 6539 diff --git a/5qmtbx9d/wandb/debug-internal.log b/5qmtbx9d/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8ff1444cb7b622a6dfb0cbc5e2e7f4d5746103bc --- /dev/null +++ b/5qmtbx9d/wandb/debug-internal.log @@ -0,0 +1,786 @@ +2025-01-07 20:44:51,022 INFO StreamThr :2230694 [internal.py:wandb_internal():85] W&B internal server running at pid: 2230694, started at: 2025-01-07 20:44:51.022446 +2025-01-07 20:44:51,023 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:44:51,036 INFO WriterThread:2230694 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/run-5qmtbx9d.wandb +2025-01-07 20:44:51,037 DEBUG SenderThread:2230694 [sender.py:send():379] send: header +2025-01-07 20:44:51,047 DEBUG SenderThread:2230694 [sender.py:send():379] send: run +2025-01-07 20:44:51,257 INFO SenderThread:2230694 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files +2025-01-07 20:44:51,257 INFO SenderThread:2230694 [sender.py:_start_run_threads():1188] run started: 5qmtbx9d with start time 1736300691.023246 +2025-01-07 20:44:51,275 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:44:51,275 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:44:51,347 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:44:51,352 DEBUG HandlerThread:2230694 [system_info.py:__init__():26] System info init +2025-01-07 20:44:51,352 DEBUG HandlerThread:2230694 [system_info.py:__init__():41] System info init done +2025-01-07 20:44:51,352 INFO HandlerThread:2230694 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:44:51,352 INFO SystemMonitor:2230694 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:44:51,352 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:44:51,352 INFO HandlerThread:2230694 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:44:51,353 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:44:51,354 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:44:51,355 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:44:51,355 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:44:51,366 DEBUG HandlerThread:2230694 [system_info.py:probe():152] Probing system +2025-01-07 20:44:51,368 DEBUG HandlerThread:2230694 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_info.py:probe():200] Probing system done +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:44:51.366285', 'startedAt': '2025-01-08T01:44:51.015220', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.647375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.354, 'min': 800.0, 'max': 4700.0}, {'current': 4499.98, 'min': 800.0, 'max': 4700.0}, {'current': 4498.377, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 606.8205108642578}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:44:51,373 INFO HandlerThread:2230694 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:44:51,373 INFO HandlerThread:2230694 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_info.py:_save_code():44] Saving code +2025-01-07 20:44:51,374 DEBUG HandlerThread:2230694 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:44:51,374 DEBUG HandlerThread:2230694 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:44:51,438 DEBUG HandlerThread:2230694 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:44:51,438 INFO HandlerThread:2230694 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:44:51,442 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:44:51,442 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:44:51,442 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:44:51,442 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:44:51,686 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/xtnom47k-wandb-metadata.json +2025-01-07 20:44:51,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:44:51,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:44:51,689 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:44:51,690 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:44:51,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:51,694 INFO HandlerThread:2230694 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/tb" + save: true +} +control { + relay_id: "2658d00ea60f408ba372987bc0ee3bb1" +} +_info { + stream_id: "5qmtbx9d" +} + +2025-01-07 20:44:51,694 DEBUG HandlerThread:2230694 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:44:51,730 INFO wandb-upload_1:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/9ejhf09c-code/FISH/eval_robot.py +2025-01-07 20:44:51,765 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/q67j575q-diff.patch +2025-01-07 20:44:51,796 DEBUG SenderThread:2230694 [sender.py:send():379] send: telemetry +2025-01-07 20:44:51,796 DEBUG SenderThread:2230694 [sender.py:send():379] send: tbrecord +2025-01-07 20:44:51,796 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:44:51,796 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 with policy live +2025-01-07 20:44:51,983 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/ixpipyy5-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH/eval_robot.py +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-metadata.json +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/diff.patch +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH +2025-01-07 20:44:52,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:53,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:54,258 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:44:54,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:55,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:56,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:56,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:57,155 DEBUG SenderThread:2230694 [sender.py:send():379] send: config +2025-01-07 20:44:57,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:58,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:44:58,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:59,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:00,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:00,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:01,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:01,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:02,261 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:02,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:03,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:04,261 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:04,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:05,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:06,262 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:06,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:06,690 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:06,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:06,788 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:07,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:08,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:09,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:10,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:11,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:12,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:12,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:13,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:14,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:15,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:16,265 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:16,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:17,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:17,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:18,265 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:18,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:19,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:20,266 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:20,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:21,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:21,690 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:21,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:22,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:22,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:23,267 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml +2025-01-07 20:45:23,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:24,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:25,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:26,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:27,695 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:28,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:28,695 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:29,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:30,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:31,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:32,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:33,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:34,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:34,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:35,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:36,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:36,691 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:36,737 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:37,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:38,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:39,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:39,798 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:40,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:41,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:42,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:43,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:44,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:45,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:45,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:46,649 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:45:46,649 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 with policy now +2025-01-07 20:45:46,650 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:45:46,650 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:45:46,650 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:45:46,651 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:45:46,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:46,898 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/zlapmflm-media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:45:47,277 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos +2025-01-07 20:45:47,568 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/enx1w92h-media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:45:47,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:48,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:48,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:49,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:50,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:51,355 DEBUG SystemMonitor:2230694 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:45:51,356 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:45:51,356 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:51,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:51,692 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:51,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:52,279 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:52,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:53,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:54,280 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:54,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:55,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:56,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:56,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:57,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:58,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:59,282 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:45:59,573 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/eahdd1sd-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:45:59,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:00,282 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:00,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:01,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:01,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:02,283 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:02,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:03,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:04,284 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:04,712 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:05,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:06,284 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:06,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:06,691 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:06,737 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:06,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:07,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:08,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:09,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:10,712 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:11,730 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:12,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:12,730 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:13,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:14,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:15,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:16,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:17,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:17,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:18,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:19,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:20,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:21,357 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:46:21,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:21,692 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:21,737 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:22,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:22,851 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:23,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:24,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:25,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:26,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:27,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:28,695 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:28,734 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:29,747 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:30,747 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:31,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:32,226 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:46:32,226 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 with policy now +2025-01-07 20:46:32,226 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:46:32,234 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:46:32,234 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:46:32,234 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:46:32,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:32,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:46:32,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:46:32,294 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:46:32,487 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/z0oizjyf-media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:46:32,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:32,764 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/kncz13fo-media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:46:33,748 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:34,238 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:34,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:34,749 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:35,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:36,696 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:36,696 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:36,751 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:37,749 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:38,294 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:38,761 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:39,749 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:39,762 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:40,295 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:40,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:41,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:42,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:43,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:44,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:45,094 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:46:45,095 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:46:45,096 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:45,096 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:46:45,297 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:46:45,297 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:46:45,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:46,297 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:46,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:47,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:48,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:49,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:50,298 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:50,382 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:50,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:51,357 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:46:51,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:51,693 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:51,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:52,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:53,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:54,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:55,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:55,799 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:56,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:57,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:58,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:59,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:00,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:01,697 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:01,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:02,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:03,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:04,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:05,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:06,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:06,693 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:06,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:06,852 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:07,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:08,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:09,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:10,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:11,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:12,698 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:12,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:13,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:14,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:15,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:16,762 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:17,109 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:47:17,109 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e7a91725641848097164.mp4 with policy now +2025-01-07 20:47:17,112 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:47:17,112 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:47:17,112 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:47:17,114 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:47:17,307 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:47:17,308 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:47:17,308 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:47:17,389 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/yjzjvrbu-media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:47:17,696 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/ypimv1a0-media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:47:17,759 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:18,126 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:18,308 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:18,759 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:19,758 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:20,761 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:21,357 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:47:21,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:21,694 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:21,763 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:22,309 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:22,759 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:23,661 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:23,758 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:24,763 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:25,763 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:26,310 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:26,762 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:27,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:28,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:28,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:29,766 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:30,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:31,312 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:47:31,571 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/b41zrsei-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:47:31,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:31,935 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:47:31,936 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:47:31,936 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:47:32,312 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:32,313 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:47:32,771 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:33,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:34,209 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:34,313 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:34,776 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:35,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:36,314 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:36,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:36,694 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:36,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:37,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:38,315 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:38,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:39,769 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:39,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:40,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:41,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:42,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:43,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:44,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:45,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:45,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:46,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:47,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:48,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:49,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:50,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:50,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:51,358 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:47:51,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:51,694 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:51,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:52,320 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:52,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:53,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:54,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:55,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:56,217 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:56,321 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:56,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:57,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:58,322 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:58,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:59,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:00,775 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:01,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:01,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:02,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:03,779 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:03,973 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:48:03,973 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:48:03,973 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 with policy now +2025-01-07 20:48:03,974 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:48:03,974 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:48:03,974 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:48:04,259 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/velqlxbr-media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:48:04,324 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:04,324 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:48:04,325 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:48:04,325 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:48:04,602 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/e26avb6z-media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:48:04,780 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:05,782 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:06,325 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:06,704 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:06,705 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:06,782 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:06,857 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:07,789 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:08,792 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:09,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:10,326 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:10,784 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:11,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:12,327 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:12,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:12,787 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:13,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:14,788 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:15,802 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:16,799 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:17,742 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:17,791 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:17,991 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:48:17,991 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:48:17,992 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:48:18,329 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:48:18,329 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:48:18,813 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:19,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:20,330 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:20,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:21,358 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:48:21,697 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:21,697 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:21,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:22,330 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:22,781 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:22,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:23,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:24,331 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:24,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:25,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:26,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:27,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:28,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:28,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:29,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:30,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:31,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:32,334 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:32,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:33,782 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:33,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:34,335 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:34,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:35,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:36,336 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:36,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:36,695 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:36,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:37,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:38,336 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:38,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:38,855 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:39,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:40,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:41,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:42,338 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:42,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:43,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:44,711 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:44,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:45,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:46,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:47,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:48,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:49,711 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:49,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:50,809 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:51,227 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:48:51,227 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 with policy now +2025-01-07 20:48:51,227 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:48:51,228 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:48:51,228 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:48:51,228 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:48:51,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:48:51,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:48:51,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:48:51,359 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:48:51,493 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/iz2umrsk-media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:48:51,700 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:51,700 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:51,756 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/2vthxn43-media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:48:51,803 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:52,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:52,807 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:53,809 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:54,813 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:54,849 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:55,806 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:56,343 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:56,811 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:57,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:58,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:59,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:00,345 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:00,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:00,824 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:01,824 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:02,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:03,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:04,831 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:05,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:06,720 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:49:06,720 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:49:06,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:06,861 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:07,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:08,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:09,053 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:09,053 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:09,054 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:09,262 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:49:09,262 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table +2025-01-07 20:49:09,473 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:49:09,530 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/f40siacd-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:09,622 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/wj745w6t-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:09,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:09,907 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:49:09,949 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:49:09,962 INFO wandb-upload_1:2230694 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqtubs6oi +2025-01-07 20:49:10,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:10,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:10,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:10,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval +2025-01-07 20:49:10,381 INFO SenderThread:2230694 [sender.py:send_request_log_artifact():1525] logged artifact run-5qmtbx9d-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxOTA4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0MDcwMg==', 'latestArtifact': None}} +2025-01-07 20:49:10,381 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:49:10,381 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 20:49:10,382 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:10,382 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,383 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,383 DEBUG SenderThread:2230694 [sender.py:send():379] send: telemetry +2025-01-07 20:49:10,383 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,387 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,387 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,387 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,388 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,388 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,388 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,388 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,388 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,389 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,389 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,389 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,389 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,390 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,390 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,390 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,390 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,390 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,391 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,391 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,391 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,392 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,392 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:49:10,555 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/wx4rvr4x-media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:10,634 INFO wandb-upload_1:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/s88a5ryk-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:10,787 INFO wandb-upload_0:2230694 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd4iga8dg +2025-01-07 20:49:10,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:11,187 INFO SenderThread:2230694 [sender.py:send_request_log_artifact():1525] logged artifact run-5qmtbx9d-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxOTEwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0MDcwNA==', 'latestArtifact': None}} +2025-01-07 20:49:11,187 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:49:11,187 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json with policy now +2025-01-07 20:49:11,188 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:11,188 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:11,188 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:11,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:11,432 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/lmo2jsol-media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:11,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:12,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:12,717 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:12,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:13,350 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml +2025-01-07 20:49:13,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:14,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:15,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:16,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:17,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:18,714 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:18,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:19,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:20,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:21,359 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:49:21,698 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:49:21,698 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:49:21,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:22,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:23,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:23,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:24,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:25,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:26,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:27,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:28,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:29,714 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:29,829 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:29,885 DEBUG SenderThread:2230694 [sender.py:send():379] send: exit +2025-01-07 20:49:29,885 INFO SenderThread:2230694 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:send_exit():588] handling runtime: 278 +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:send_exit():594] send defer +2025-01-07 20:49:29,886 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,886 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:49:29,886 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:49:29,886 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,886 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:49:29,887 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,887 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:49:29,887 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:49:29,887 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,887 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:49:29,887 INFO HandlerThread:2230694 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:49:29,887 DEBUG SystemMonitor:2230694 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:49:29,887 DEBUG SystemMonitor:2230694 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:49:29,887 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:49:29,888 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:49:29,941 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:49:29,941 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:49:29,941 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:49:29,941 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,942 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:49:29,942 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:49:29,942 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:49:29,942 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,942 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:49:29,943 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,943 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:49:29,943 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:49:29,943 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,943 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:49:30,356 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:32,357 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:36,061 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:36,061 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:36,061 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:49:36,062 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,063 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:49:36,063 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:49:36,063 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:36,063 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:36,063 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:36,063 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,063 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:36,063 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:49:36,064 DEBUG SenderThread:2230694 [sender.py:send():379] send: summary +2025-01-07 20:49:36,064 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:36,065 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:49:36,065 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,065 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:49:36,065 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:49:36,065 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:36,065 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,065 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:49:36,066 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,066 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:49:36,114 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:49:36,114 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,114 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:49:36,115 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,115 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:49:36,115 INFO SenderThread:2230694 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:49:36,117 INFO SenderThread:2230694 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:49:36,119 INFO SenderThread:2230694 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:49:36,129 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:49:36,130 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,130 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:49:36,130 DEBUG SenderThread:2230694 [sender.py:send():379] send: artifact +2025-01-07 20:49:36,347 INFO SenderThread:2230694 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:49:36,347 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,347 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:49:36,347 INFO SenderThread:2230694 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:49:36,358 INFO SenderThread:2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml config.yaml +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json wandb-summary.json +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt requirements.txt +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:36,362 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log output.log +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/diff.patch diff.patch +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:49:36,365 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:49:36,365 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:49:36,365 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:49:36,368 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,368 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:49:36,368 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:49:36,542 INFO wandb-upload_1:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:36,552 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml +2025-01-07 20:49:36,584 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt +2025-01-07 20:49:36,611 INFO wandb-upload_4:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:36,645 INFO wandb-upload_3:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/4tovxwq2-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:36,846 INFO Thread-11 (_thread_body):2230694 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:49:36,846 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,846 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:49:36,846 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,846 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:49:36,846 INFO SenderThread:2230694 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:49:36,848 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:49:36,848 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,848 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:49:36,848 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,848 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:49:36,848 INFO SenderThread:2230694 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:49:36,887 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:37,126 INFO SenderThread:2230694 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:49:37,126 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:49:37,126 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:37,126 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:37,126 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:49:37,126 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:37,126 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:49:37,127 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:49:37,127 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:37,127 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:49:37,127 DEBUG SenderThread:2230694 [sender.py:send():379] send: final +2025-01-07 20:49:37,127 DEBUG SenderThread:2230694 [sender.py:send():379] send: footer +2025-01-07 20:49:37,127 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:37,127 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:49:37,128 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:37,128 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:37,128 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:49:37,128 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:49:37,128 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:37,130 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:49:37,130 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:49:37,131 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:37,182 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:37,182 INFO MainThread:2230694 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:49:37,183 INFO MainThread:2230694 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:49:37,183 INFO MainThread:2230694 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:49:37,183 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:49:37,183 INFO HandlerThread:2230694 [handler.py:finish():882] shutting down handler +2025-01-07 20:49:38,130 INFO WriterThread:2230694 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/run-5qmtbx9d.wandb +2025-01-07 20:49:38,182 INFO SenderThread:2230694 [sender.py:finish():1615] shutting down sender +2025-01-07 20:49:38,182 INFO SenderThread:2230694 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:49:38,182 INFO SenderThread:2230694 [file_pusher.py:join():175] waiting for file pusher diff --git a/5qmtbx9d/wandb/debug.log b/5qmtbx9d/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1f0c4eb83723bde7a5aa5649dd68c0cadd98df27 --- /dev/null +++ b/5qmtbx9d/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:44:51,016 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:44:51,016 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Configure stats pid to 2230506 +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/settings +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug.log +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug-internal.log +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d'} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():619] starting backend +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():623] setting up manager +2025-01-07 20:44:51,021 INFO MainThread:2230506 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:44:51,023 INFO MainThread:2230506 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:44:51,036 INFO MainThread:2230506 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:44:51,045 INFO MainThread:2230506 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:44:51,274 INFO MainThread:2230506 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:44:51,342 INFO MainThread:2230506 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:44:51,342 INFO MainThread:2230506 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:44:51,689 INFO MainThread:2230506 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:44:51,690 INFO MainThread:2230506 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:44:51,690 INFO MainThread:2230506 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:44:51,690 INFO MainThread:2230506 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:44:51,694 INFO MainThread:2230506 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:44:51,694 INFO MainThread:2230506 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/tb, True +2025-01-07 20:44:57,155 INFO MainThread:2230506 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:49:38,184 WARNING MsgRouterThr:2230506 [router.py:message_loop():77] message_loop has been closed diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH/eval_robot.py b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..366e616166a3efdb5d921c7f08e425700e9b36d6 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736300691 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/diff.patch b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c0161c26d9b20c7c85242592d6833c2df9767742 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/diff.patch @@ -0,0 +1,20 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..aa5b9b9 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,13 +134,13 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' +-# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e8064c158f73fa409a430f791fd90332683ba2b5 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[55, 55, 1.0, 0.9980408548312694, 1, 0.9407418443871209, 1, 0.9488667286034733, 1, 1, 1]]} \ No newline at end of file diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e2363a734237f10dab2acb709fd7ef1a2e3ba6a --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:310b504403e8beda16b23ccfe0eebdf0935145269ae9a1ba2bd2c15f13727a16 +size 1146404 diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c90ff384afc72c15b16f04b05fbbfd84bf1dfc97 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c060e05047508277ede7722f7f3a0797eb47f21deae6e589348459a43ecc7931 +size 1209168 diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d1ce764cb547a5adafdf823256b00bc6911c7f6 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7a9172564184809716447311f702d7b29617121735a2e139cc8d1d58b2b922d +size 1181538 diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60efe2058d01191c9920ffc6d68c7c5ee96dfd79 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f11c37a5819a4ad2728beb9c63a371103173857454e736393d23a2544f058d4c +size 1194205 diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfd5c897fb4c082fa713684b227a72d247fdf27f --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34d4110b1ed92fa8405fd4ab00f15027d790141e6db4d43d1a07c96206eb1c63 +size 1209669 diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1f87c49062a322916d52e16cf36d37458a208e27 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736300697.156025]: resetting environment +[INFO] [1736300697.160578]: cleared current plan +[INFO] [1736300697.162024]: moving to home +[INFO] [1736300698.164222]: reached home +[INFO] [1736300698.165577]: reset action history +[INFO] [1736300700.874087]: environment reset +Starting episode 0 +[INFO] [1736300700.876204]: resetting environment +[INFO] [1736300700.879044]: cleared current plan +[INFO] [1736300700.879805]: moving to home +[INFO] [1736300701.881274]: reached home +[INFO] [1736300701.882873]: reset action history +[INFO] [1736300704.593837]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736300714.995302]: resetting environment +[INFO] [1736300714.998703]: cleared current plan +[INFO] [1736300714.999581]: moving to home +[INFO] [1736300716.001279]: reached home +[INFO] [1736300716.002532]: reset action history +[INFO] [1736300718.711104]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/episode_rosbags/episode_0_2025-01-07-20-45-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2230506.log +started roslaunch server http://158.130.50.37:35059/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2230797] +process[print_text-2]: started with pid [2230798] +process[rosbag_record-3]: started with pid [2230821] +started bagging! +[INFO] [1736300745.710956]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736300746.443021]: Stored episode 1. +[INFO] [1736300746.443229]: Saving video... +[INFO] [1736300746.635333]: Video saved! +Starting episode 1 +[INFO] [1736300746.653457]: resetting environment +[INFO] [1736300746.660882]: cleared current plan +[INFO] [1736300746.661859]: moving to home +[INFO] [1736300750.465104]: reached home +[INFO] [1736300750.465583]: reset action history +[INFO] [1736300753.170730]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300758.880056]: resetting environment +[INFO] [1736300758.884127]: cleared current plan +[INFO] [1736300758.886644]: moving to home +[INFO] [1736300759.889277]: reached home +[INFO] [1736300759.893520]: reset action history +[INFO] [1736300762.604549]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/episode_rosbags/episode_1_2025-01-07-20-46-02.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2230506.log +started roslaunch server http://158.130.50.37:38379/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2230954] +process[print_text-5]: started with pid [2230955] +process[rosbag_record-6]: started with pid [2230956] +started bagging! +[INFO] [1736300791.226288]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit[print_text-5] killing on exit +[INFO] [1736300791.988106]: Stored episode 2. +[INFO] [1736300791.989653]: Saving video... +[INFO] [1736300792.205823]: Video saved! +Starting episode 2 +[INFO] [1736300792.229818]: resetting environment +[INFO] [1736300792.236851]: cleared current plan +[INFO] [1736300792.238127]: moving to home +[INFO] [1736300795.941826]: reached home +[INFO] [1736300795.943595]: reset action history +[INFO] [1736300798.716942]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300804.340442]: resetting environment +[INFO] [1736300804.351130]: cleared current plan +[INFO] [1736300804.352237]: moving to home +[INFO] [1736300805.353962]: reached home +[INFO] [1736300805.355935]: reset action history +[INFO] [1736300808.074966]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/episode_rosbags/episode_2_2025-01-07-20-46-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2230506.log +started roslaunch server http://158.130.50.37:39513/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2231092] +process[print_text-8]: started with pid [2231093] +process[rosbag_record-9]: started with pid [2231094] +started bagging! +[INFO] [1736300836.088473]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736300836.862779]: Stored episode 3. +[INFO] [1736300836.864586]: Saving video... +[INFO] [1736300837.080536]: Video saved! +Starting episode 3 +[INFO] [1736300837.118798]: resetting environment +[INFO] [1736300837.125989]: cleared current plan +[INFO] [1736300837.126264]: moving to home +[INFO] [1736300840.933831]: reached home +[INFO] [1736300840.934717]: reset action history +[INFO] [1736300843.660967]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300851.148459]: resetting environment +[INFO] [1736300851.154896]: cleared current plan +[INFO] [1736300851.156806]: moving to home +[INFO] [1736300852.158985]: reached home +[INFO] [1736300852.163762]: reset action history +[INFO] [1736300854.889781]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/episode_rosbags/episode_3_2025-01-07-20-47-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2230506.log +started roslaunch server http://158.130.50.37:42281/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2231226] +process[print_text-11]: started with pid [2231227] +process[rosbag_record-12]: started with pid [2231228] +started bagging! +For topic gripper_width: timestamp difference is 103100901 for nearest: 1736300870769327526 - target: 1736300870666226625 at idx 222 +For topic gripper_width: timestamp difference is 106740948 for nearest: 1736300875169323679 - target: 1736300875062582731 at idx 249 +For topic gripper_width: timestamp difference is 107896706 for nearest: 1736300876569324633 - target: 1736300876461427927 at idx 249 +[INFO] [1736300882.980351]: Storing episode... +[print_text-11] killing on exit +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1736300883.713939]: Stored episode 4. +[INFO] [1736300883.721574]: Saving video... +[INFO] [1736300883.927690]: Video saved! +Starting episode 4 +[INFO] [1736300883.976028]: resetting environment +[INFO] [1736300883.985331]: cleared current plan +[INFO] [1736300883.987026]: moving to home +[INFO] [1736300887.704285]: reached home +[INFO] [1736300887.706740]: reset action history +[INFO] [1736300890.484603]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300897.741840]: resetting environment +[INFO] [1736300897.754483]: cleared current plan +[INFO] [1736300897.760657]: moving to home +[INFO] [1736300898.771809]: reached home +[INFO] [1736300898.775058]: reset action history +[INFO] [1736300901.507777]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/episode_rosbags/episode_4_2025-01-07-20-48-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2230506.log +started roslaunch server http://158.130.50.37:40089/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2231364] +process[print_text-14]: started with pid [2231365] +process[rosbag_record-15]: started with pid [2231366] +started bagging! +For topic gripper_width: timestamp difference is 101406457 for nearest: 1736300909969323995 - target: 1736300909867917538 at idx 113 +For topic gripper_width: timestamp difference is 101521332 for nearest: 1736300910169278223 - target: 1736300910067756891 at idx 115 +For topic gripper_width: timestamp difference is 103532054 for nearest: 1736300912569323756 - target: 1736300912465791702 at idx 148 +For topic gripper_width: timestamp difference is 103697766 for nearest: 1736300912769325246 - target: 1736300912665627480 at idx 150 +For topic gripper_width: timestamp difference is 104725478 for nearest: 1736300913969365237 - target: 1736300913864639759 at idx 170 +For topic gripper_width: timestamp difference is 105974098 for nearest: 1736300915569327970 - target: 1736300915463353872 at idx 190 +For topic gripper_width: timestamp difference is 110025361 for nearest: 1736300920602657512 - target: 1736300920492632151 at idx 249 +[INFO] [1736300930.159136]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736300930.939366]: Stored episode 5. +[INFO] [1736300930.944683]: Saving video... +[INFO] [1736300931.186953]: Video saved! +[INFO] [1736300931.228069]: resetting environment +[INFO] [1736300931.243566]: cleared current plan +[INFO] [1736300931.247611]: moving to home +[INFO] [1736300934.959472]: reached home +[INFO] [1736300934.967095]: reset action history +[INFO] [1736300937.744494]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-metadata.json b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..53de4dbe6efb21a7e2265d1a5254730ea21840f8 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:44:51.366285", + "startedAt": "2025-01-08T01:44:51.015220", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.647375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.354, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.98, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.377, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 606.8205108642578 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..45d6fd1c010c49b101e5282c869546e95ad50d3f --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "310b504403e8beda16b23ccfe0eebdf0935145269ae9a1ba2bd2c15f13727a16", "size": 1146404, "path": "media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4"}, "global_step": 665, "_timestamp": 1736300949.7969847, "_runtime": 258.7737386226654, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "c060e05047508277ede7722f7f3a0797eb47f21deae6e589348459a43ecc7931", "size": 1209168, "path": "media/videos/eval/1_eval_1_c060e05047508277ede7.mp4"}, "eval/num_success": 5.0, "episode": 55.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "e7a9172564184809716447311f702d7b29617121735a2e139cc8d1d58b2b922d", "size": 1181538, "path": "media/videos/eval/2_eval_2_e7a91725641848097164.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f11c37a5819a4ad2728beb9c63a371103173857454e736393d23a2544f058d4c", "size": 1194205, "path": "media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "34d4110b1ed92fa8405fd4ab00f15027d790141e6db4d43d1a07c96206eb1c63", "size": 1209669, "path": "media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://p2c8egnja51h6a5j3u7mpah3mv15m76xpfxee2pc7kxbm93np7m7s9tjt0exi52jehv5qk46esu9zdg3di8d0qd1dg4jx0bghg5nf5uk9lq7ryzw2kdq33cqf2ptu2bd/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://k8yac8b5od20y3ldjzcjpfrnjq8f9g6gxd6y3nplmwvsjjyq4iikwrti9mncncjel1ja977gmkqsq4ysz2fymh4di56x24lhklwikv0ir0u9ptvn2iqadnpemc2hdv7t:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 55, "total_num_episodes": 55, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9980408548312694, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9407418443871209, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9488667286034733, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "037e05873b3b41256eca73c3a9b0608fced19a4b133260e547c17dedc3531d7d", "size": 420, "artifact_path": "wandb-client-artifact://2feyozgulv8gspl2z4w2ho6umw95vsagksfqe9mcx6lk9n149p8hvfn2mcgyd0g2ts9qq7ggghvcitnxgosnjb8gzejaw8hof9girg82larhvpdqkns9yciuxoqnxdtk/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ffj9slycbeqq84rdnkc8cqrz19hsadc9uhm7i1yxvuxa09ry7x7whpvmkvkzqewxl3ga6pcgsrcviei8gzf4batylwf8eqnq4a89v8rjsq57vraz6exi7oagokjwlkkc:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 278}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9980408549308777, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.9407418370246887, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9488667249679565, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug-internal.log b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8ff1444cb7b622a6dfb0cbc5e2e7f4d5746103bc --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug-internal.log @@ -0,0 +1,786 @@ +2025-01-07 20:44:51,022 INFO StreamThr :2230694 [internal.py:wandb_internal():85] W&B internal server running at pid: 2230694, started at: 2025-01-07 20:44:51.022446 +2025-01-07 20:44:51,023 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:44:51,036 INFO WriterThread:2230694 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/run-5qmtbx9d.wandb +2025-01-07 20:44:51,037 DEBUG SenderThread:2230694 [sender.py:send():379] send: header +2025-01-07 20:44:51,047 DEBUG SenderThread:2230694 [sender.py:send():379] send: run +2025-01-07 20:44:51,257 INFO SenderThread:2230694 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files +2025-01-07 20:44:51,257 INFO SenderThread:2230694 [sender.py:_start_run_threads():1188] run started: 5qmtbx9d with start time 1736300691.023246 +2025-01-07 20:44:51,275 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:44:51,275 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:44:51,347 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:44:51,352 DEBUG HandlerThread:2230694 [system_info.py:__init__():26] System info init +2025-01-07 20:44:51,352 DEBUG HandlerThread:2230694 [system_info.py:__init__():41] System info init done +2025-01-07 20:44:51,352 INFO HandlerThread:2230694 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:44:51,352 INFO SystemMonitor:2230694 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:44:51,352 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:44:51,352 INFO HandlerThread:2230694 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:44:51,353 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:44:51,354 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:44:51,355 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:44:51,355 INFO SystemMonitor:2230694 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:44:51,366 DEBUG HandlerThread:2230694 [system_info.py:probe():152] Probing system +2025-01-07 20:44:51,368 DEBUG HandlerThread:2230694 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_info.py:probe():200] Probing system done +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:44:51.366285', 'startedAt': '2025-01-08T01:44:51.015220', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.647375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.354, 'min': 800.0, 'max': 4700.0}, {'current': 4499.98, 'min': 800.0, 'max': 4700.0}, {'current': 4498.377, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 606.8205108642578}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:44:51,373 INFO HandlerThread:2230694 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:44:51,373 INFO HandlerThread:2230694 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:44:51,373 DEBUG HandlerThread:2230694 [system_info.py:_save_code():44] Saving code +2025-01-07 20:44:51,374 DEBUG HandlerThread:2230694 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:44:51,374 DEBUG HandlerThread:2230694 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:44:51,438 DEBUG HandlerThread:2230694 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:44:51,438 INFO HandlerThread:2230694 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:44:51,442 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:44:51,442 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:44:51,442 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:44:51,442 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:44:51,686 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/xtnom47k-wandb-metadata.json +2025-01-07 20:44:51,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:44:51,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:44:51,689 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:44:51,690 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:44:51,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:51,694 INFO HandlerThread:2230694 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/tb" + save: true +} +control { + relay_id: "2658d00ea60f408ba372987bc0ee3bb1" +} +_info { + stream_id: "5qmtbx9d" +} + +2025-01-07 20:44:51,694 DEBUG HandlerThread:2230694 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:44:51,730 INFO wandb-upload_1:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/9ejhf09c-code/FISH/eval_robot.py +2025-01-07 20:44:51,765 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/q67j575q-diff.patch +2025-01-07 20:44:51,796 DEBUG SenderThread:2230694 [sender.py:send():379] send: telemetry +2025-01-07 20:44:51,796 DEBUG SenderThread:2230694 [sender.py:send():379] send: tbrecord +2025-01-07 20:44:51,796 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:44:51,796 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 with policy live +2025-01-07 20:44:51,983 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/ixpipyy5-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt +2025-01-07 20:44:52,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH/eval_robot.py +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-metadata.json +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/diff.patch +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code +2025-01-07 20:44:52,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH +2025-01-07 20:44:52,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:53,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:54,258 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:44:54,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:55,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:56,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:56,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:57,155 DEBUG SenderThread:2230694 [sender.py:send():379] send: config +2025-01-07 20:44:57,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:58,259 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:44:58,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:59,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:00,260 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:00,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:01,690 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:01,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:02,261 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:02,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:03,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:04,261 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:04,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:05,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:06,262 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:06,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:06,690 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:06,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:06,788 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:07,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:08,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:09,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:10,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:11,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:12,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:12,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:13,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:14,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:15,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:16,265 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:16,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:17,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:17,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:18,265 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:18,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:19,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:20,266 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:20,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:21,689 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:21,690 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:21,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:22,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:22,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:23,267 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml +2025-01-07 20:45:23,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:24,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:25,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:26,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:27,695 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:28,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:28,695 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:29,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:30,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:31,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:32,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:33,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:34,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:34,701 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:35,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:36,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:36,691 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:36,737 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:37,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:38,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:39,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:39,798 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:40,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:41,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:42,705 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:43,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:44,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:45,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:45,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:46,649 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:45:46,649 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 with policy now +2025-01-07 20:45:46,650 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:45:46,650 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:45:46,650 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:45:46,651 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:45:46,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:46,898 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/zlapmflm-media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:45:47,277 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:45:47,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos +2025-01-07 20:45:47,568 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/enx1w92h-media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:45:47,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:48,278 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:48,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:49,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:50,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:51,355 DEBUG SystemMonitor:2230694 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:45:51,356 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:45:51,356 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:51,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:45:51,692 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:45:51,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:52,279 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:52,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:53,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:54,280 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:45:54,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:55,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:56,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:45:56,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:57,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:58,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:45:59,282 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:45:59,573 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/eahdd1sd-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:45:59,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:00,282 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:00,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:01,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:01,708 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:02,283 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:02,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:03,709 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:04,284 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:04,712 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:05,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:06,284 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:06,691 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:06,691 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:06,737 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:06,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:07,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:08,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:09,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:10,712 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:11,730 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:12,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:12,730 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:13,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:14,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:15,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:16,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:17,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:17,731 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:18,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:19,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:20,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:21,357 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:46:21,692 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:21,692 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:21,737 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:22,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:22,851 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:23,732 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:24,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:25,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:26,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:27,733 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:28,695 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:28,734 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:29,747 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:30,747 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:31,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:32,226 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:46:32,226 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 with policy now +2025-01-07 20:46:32,226 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:46:32,234 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:46:32,234 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:46:32,234 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:46:32,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:32,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:46:32,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:46:32,294 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:46:32,487 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/z0oizjyf-media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:46:32,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:32,764 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/kncz13fo-media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:46:33,748 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:34,238 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:34,293 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:34,749 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:35,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:36,696 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:36,696 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:36,751 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:37,749 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:38,294 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:38,761 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:39,749 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:39,762 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:40,295 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:40,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:41,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:42,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:43,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:44,750 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:45,094 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:46:45,095 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:46:45,096 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:45,096 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:46:45,297 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:46:45,297 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:46:45,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:46,297 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:46,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:47,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:48,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:49,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:50,298 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:46:50,382 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:50,752 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:51,357 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:46:51,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:46:51,693 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:46:51,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:52,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:53,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:54,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:55,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:55,799 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:46:56,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:57,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:58,753 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:46:59,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:00,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:01,697 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:01,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:02,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:03,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:04,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:05,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:06,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:06,693 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:06,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:06,852 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:07,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:08,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:09,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:10,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:11,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:12,698 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:12,755 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:13,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:14,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:15,756 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:16,762 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:17,109 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:47:17,109 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e7a91725641848097164.mp4 with policy now +2025-01-07 20:47:17,112 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:47:17,112 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:47:17,112 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:47:17,114 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:47:17,307 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:47:17,308 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:47:17,308 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:47:17,389 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/yjzjvrbu-media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:47:17,696 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/ypimv1a0-media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:47:17,759 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:18,126 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:18,308 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:18,759 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:19,758 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:20,761 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:21,357 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:47:21,693 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:21,694 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:21,763 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:22,309 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:22,759 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:23,661 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:23,758 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:24,763 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:25,763 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:26,310 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:26,762 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:27,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:28,707 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:28,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:29,766 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:30,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:31,312 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:47:31,571 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/b41zrsei-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:47:31,765 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:31,935 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:47:31,936 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:47:31,936 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:47:32,312 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:32,313 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:47:32,771 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:33,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:34,209 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:34,313 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:34,776 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:35,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:36,314 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:36,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:36,694 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:36,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:37,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:38,315 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:38,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:39,769 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:39,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:40,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:41,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:42,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:43,772 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:44,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:45,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:45,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:46,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:47,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:48,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:49,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:50,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:50,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:51,358 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:47:51,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:47:51,694 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:47:51,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:52,320 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:52,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:53,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:54,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:55,773 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:56,217 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:47:56,321 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:56,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:57,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:58,322 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:47:58,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:47:59,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:00,775 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:01,702 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:01,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:02,774 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:03,779 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:03,973 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:48:03,973 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:48:03,973 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 with policy now +2025-01-07 20:48:03,974 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:48:03,974 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:48:03,974 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:48:04,259 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/velqlxbr-media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:48:04,324 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:04,324 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:48:04,325 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:48:04,325 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:48:04,602 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/e26avb6z-media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:48:04,780 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:05,782 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:06,325 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:06,704 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:06,705 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:06,782 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:06,857 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:07,789 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:08,792 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:09,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:10,326 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:10,784 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:11,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:12,327 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:12,706 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:12,787 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:13,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:14,788 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:15,802 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:16,799 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:17,742 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:17,791 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:17,991 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:48:17,991 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:48:17,992 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:48:18,329 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:48:18,329 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:48:18,813 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:19,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:20,330 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:20,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:21,358 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:48:21,697 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:21,697 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:21,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:22,330 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:22,781 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:22,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:23,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:24,331 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:24,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:25,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:26,794 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:27,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:28,710 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:28,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:29,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:30,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:31,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:32,334 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:32,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:33,782 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:33,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:34,335 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:34,795 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:35,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:36,336 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:36,694 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:36,695 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:36,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:37,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:38,336 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:38,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:38,855 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:39,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:40,796 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:41,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:42,338 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:42,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:43,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:44,711 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:44,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:45,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:46,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:47,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:48,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:49,711 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:49,797 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:50,809 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:51,227 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:48:51,227 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 with policy now +2025-01-07 20:48:51,227 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:48:51,228 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:48:51,228 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:48:51,228 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:48:51,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:48:51,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:48:51,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval +2025-01-07 20:48:51,359 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:48:51,493 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/iz2umrsk-media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:48:51,700 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:48:51,700 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:48:51,756 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/2vthxn43-media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:48:51,803 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:52,342 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:52,807 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:53,809 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:54,813 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:54,849 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:48:55,806 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:56,343 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:48:56,811 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:57,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:58,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:48:59,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:00,345 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:00,793 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:00,824 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:01,824 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:02,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:03,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:04,831 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:05,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:06,720 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:49:06,720 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:49:06,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:06,861 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:07,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:08,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:09,053 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:09,053 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:09,054 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:09,262 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:49:09,262 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval +2025-01-07 20:49:09,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table +2025-01-07 20:49:09,473 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:49:09,530 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/f40siacd-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:09,622 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/wj745w6t-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:09,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:09,907 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:49:09,949 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:49:09,962 INFO wandb-upload_1:2230694 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqtubs6oi +2025-01-07 20:49:10,348 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:10,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:10,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:10,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval +2025-01-07 20:49:10,381 INFO SenderThread:2230694 [sender.py:send_request_log_artifact():1525] logged artifact run-5qmtbx9d-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxOTA4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0MDcwMg==', 'latestArtifact': None}} +2025-01-07 20:49:10,381 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:49:10,381 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 20:49:10,382 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:10,382 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,383 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,383 DEBUG SenderThread:2230694 [sender.py:send():379] send: telemetry +2025-01-07 20:49:10,383 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,387 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,387 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,387 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,388 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,388 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,388 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,388 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,388 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,389 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,389 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,389 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,389 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,390 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,390 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,390 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,390 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,390 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,391 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,391 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,391 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:10,392 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:10,392 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:49:10,555 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/wx4rvr4x-media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:10,634 INFO wandb-upload_1:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/s88a5ryk-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:10,787 INFO wandb-upload_0:2230694 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd4iga8dg +2025-01-07 20:49:10,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:11,187 INFO SenderThread:2230694 [sender.py:send_request_log_artifact():1525] logged artifact run-5qmtbx9d-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxOTEwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0MDcwNA==', 'latestArtifact': None}} +2025-01-07 20:49:11,187 DEBUG SenderThread:2230694 [sender.py:send():379] send: files +2025-01-07 20:49:11,187 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json with policy now +2025-01-07 20:49:11,188 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:11,188 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:11,188 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:11,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:11,432 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/lmo2jsol-media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:11,825 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:12,349 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:12,717 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:12,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:13,350 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml +2025-01-07 20:49:13,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:14,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:15,826 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:16,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:17,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:18,714 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:18,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:19,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:20,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:21,359 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:49:21,698 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:49:21,698 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:49:21,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:22,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:23,754 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:23,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:24,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:25,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:26,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:27,827 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:28,828 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:29,714 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:29,829 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:29,885 DEBUG SenderThread:2230694 [sender.py:send():379] send: exit +2025-01-07 20:49:29,885 INFO SenderThread:2230694 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:send_exit():588] handling runtime: 278 +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:send_exit():594] send defer +2025-01-07 20:49:29,886 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,886 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:49:29,886 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:49:29,886 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:49:29,886 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,886 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:49:29,887 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,887 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:49:29,887 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:49:29,887 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,887 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:49:29,887 INFO HandlerThread:2230694 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:49:29,887 DEBUG SystemMonitor:2230694 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:49:29,887 DEBUG SystemMonitor:2230694 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:49:29,887 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:49:29,888 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:49:29,941 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:49:29,941 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:49:29,941 INFO HandlerThread:2230694 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:49:29,941 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,942 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:49:29,942 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:49:29,942 DEBUG SenderThread:2230694 [sender.py:send():379] send: stats +2025-01-07 20:49:29,942 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,942 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:49:29,943 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:29,943 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:49:29,943 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:49:29,943 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:29,943 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:49:30,356 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:32,357 INFO Thread-12 :2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:36,061 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:36,061 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:36,061 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:49:36,062 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,063 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:49:36,063 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:49:36,063 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:36,063 DEBUG SenderThread:2230694 [sender.py:send():379] send: history +2025-01-07 20:49:36,063 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:49:36,063 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,063 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:36,063 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:49:36,064 DEBUG SenderThread:2230694 [sender.py:send():379] send: summary +2025-01-07 20:49:36,064 INFO SenderThread:2230694 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:49:36,065 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:49:36,065 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,065 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:49:36,065 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:49:36,065 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:49:36,065 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:49:36,065 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,065 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:49:36,066 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,066 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:49:36,114 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:49:36,114 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,114 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:49:36,115 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,115 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:49:36,115 INFO SenderThread:2230694 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:49:36,117 INFO SenderThread:2230694 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:49:36,119 INFO SenderThread:2230694 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:49:36,129 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:49:36,130 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,130 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:49:36,130 DEBUG SenderThread:2230694 [sender.py:send():379] send: artifact +2025-01-07 20:49:36,347 INFO SenderThread:2230694 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:49:36,347 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,347 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:49:36,347 INFO SenderThread:2230694 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:49:36,358 INFO SenderThread:2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml config.yaml +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json wandb-summary.json +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt requirements.txt +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:49:36,359 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:36,362 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log output.log +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/diff.patch diff.patch +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/2_eval_2_e7a91725641848097164.mp4 media/videos/eval/2_eval_2_e7a91725641848097164.mp4 +2025-01-07 20:49:36,364 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 media/videos/eval/3_eval_3_f11c37a5819a4ad2728b.mp4 +2025-01-07 20:49:36,365 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 media/videos/eval/4_eval_4_34d4110b1ed92fa8405f.mp4 +2025-01-07 20:49:36,365 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 media/videos/eval/1_eval_1_c060e05047508277ede7.mp4 +2025-01-07 20:49:36,365 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 media/videos/eval/0_eval_0_310b504403e8beda16b2.mp4 +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json media/table/eval/total_success_rate_ci_6_037e05873b3b41256eca.table.json +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:49:36,368 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,368 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:49:36,368 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:49:36,368 INFO SenderThread:2230694 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:49:36,542 INFO wandb-upload_1:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/wandb-summary.json +2025-01-07 20:49:36,552 INFO wandb-upload_0:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/config.yaml +2025-01-07 20:49:36,584 INFO wandb-upload_2:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/requirements.txt +2025-01-07 20:49:36,611 INFO wandb-upload_4:2230694 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/files/output.log +2025-01-07 20:49:36,645 INFO wandb-upload_3:2230694 [upload_job.py:push():130] Uploaded file /tmp/tmpb2aodytlwandb/4tovxwq2-events.out.tfevents.1736300691.leonmkim-ROG-Strix-G15CS-G15CS.2230506.0 +2025-01-07 20:49:36,846 INFO Thread-11 (_thread_body):2230694 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:49:36,846 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,846 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:49:36,846 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,846 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:49:36,846 INFO SenderThread:2230694 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:49:36,848 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:49:36,848 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:36,848 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:49:36,848 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:36,848 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:49:36,848 INFO SenderThread:2230694 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:49:36,887 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:37,126 INFO SenderThread:2230694 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:49:37,126 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:49:37,126 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:37,126 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:37,126 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:49:37,126 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:37,126 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:49:37,127 INFO SenderThread:2230694 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:49:37,127 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:49:37,127 INFO HandlerThread:2230694 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:49:37,127 DEBUG SenderThread:2230694 [sender.py:send():379] send: final +2025-01-07 20:49:37,127 DEBUG SenderThread:2230694 [sender.py:send():379] send: footer +2025-01-07 20:49:37,127 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: defer +2025-01-07 20:49:37,127 INFO SenderThread:2230694 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:49:37,128 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:37,128 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:37,128 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:49:37,128 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:49:37,128 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:49:37,130 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:49:37,130 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:49:37,131 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:37,182 DEBUG SenderThread:2230694 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:49:37,182 INFO MainThread:2230694 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:49:37,183 INFO MainThread:2230694 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:49:37,183 INFO MainThread:2230694 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:49:37,183 DEBUG HandlerThread:2230694 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:49:37,183 INFO HandlerThread:2230694 [handler.py:finish():882] shutting down handler +2025-01-07 20:49:38,130 INFO WriterThread:2230694 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/run-5qmtbx9d.wandb +2025-01-07 20:49:38,182 INFO SenderThread:2230694 [sender.py:finish():1615] shutting down sender +2025-01-07 20:49:38,182 INFO SenderThread:2230694 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:49:38,182 INFO SenderThread:2230694 [file_pusher.py:join():175] waiting for file pusher diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug.log b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1f0c4eb83723bde7a5aa5649dd68c0cadd98df27 --- /dev/null +++ b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:44:51,016 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:44:51,016 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Configure stats pid to 2230506 +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/settings +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug.log +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/logs/debug-internal.log +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d'} +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():619] starting backend +2025-01-07 20:44:51,017 INFO MainThread:2230506 [wandb_init.py:init():623] setting up manager +2025-01-07 20:44:51,021 INFO MainThread:2230506 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:44:51,023 INFO MainThread:2230506 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:44:51,036 INFO MainThread:2230506 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:44:51,045 INFO MainThread:2230506 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:44:51,274 INFO MainThread:2230506 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:44:51,342 INFO MainThread:2230506 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:44:51,342 INFO MainThread:2230506 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:44:51,689 INFO MainThread:2230506 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:44:51,690 INFO MainThread:2230506 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:44:51,690 INFO MainThread:2230506 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:44:51,690 INFO MainThread:2230506 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:44:51,694 INFO MainThread:2230506 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:44:51,694 INFO MainThread:2230506 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5qmtbx9d/tb, True +2025-01-07 20:44:57,155 INFO MainThread:2230506 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:49:38,184 WARNING MsgRouterThr:2230506 [router.py:message_loop():77] message_loop has been closed diff --git a/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/run-5qmtbx9d.wandb b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/run-5qmtbx9d.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f5e8b82a3414b39bfe12c1e2d809112ae2893908 Binary files /dev/null and b/5qmtbx9d/wandb/run-20250107_204451-5qmtbx9d/run-5qmtbx9d.wandb differ diff --git a/8qorj8l7/.hydra/config.yaml b/8qorj8l7/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/8qorj8l7/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/8qorj8l7/.hydra/hydra.yaml b/8qorj8l7/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8508f20b6d712a161cdb6e637e8e1c5c549f5cef --- /dev/null +++ b/8qorj8l7/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/8qorj8l7 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/8qorj8l7/.hydra/overrides.yaml b/8qorj8l7/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/8qorj8l7/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/8qorj8l7/eval_policy.log b/8qorj8l7/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..b01f3997893c723d981f1cc7a67bdc76b05719cc --- /dev/null +++ b/8qorj8l7/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 17:18:41,581][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:18:41,585][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 17:18:45,149][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/9jw2ghih/.hydra/config.yaml b/9jw2ghih/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/9jw2ghih/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/9jw2ghih/.hydra/hydra.yaml b/9jw2ghih/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23a16eacd983ef00f96fc103950d87cea8c8c7ee --- /dev/null +++ b/9jw2ghih/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/9jw2ghih/.hydra/overrides.yaml b/9jw2ghih/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/9jw2ghih/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/9jw2ghih/episode_rosbags/aligned_depth_to_color_K.npy b/9jw2ghih/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/9jw2ghih/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/9jw2ghih/episode_rosbags/cam_tf_world.npy b/9jw2ghih/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/9jw2ghih/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/9jw2ghih/episode_rosbags/color_K.npy b/9jw2ghih/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/9jw2ghih/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/9jw2ghih/episode_rosbags/depth_K.npy b/9jw2ghih/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/9jw2ghih/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/9jw2ghih/eval_robot.log b/9jw2ghih/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..3f9c9e2c0639d54b386eb689f5f325b2feb18bbb --- /dev/null +++ b/9jw2ghih/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:40:50,421][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:40:50,424][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/9jw2ghih/eval_video/0_eval.mp4 b/9jw2ghih/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d36c06c0db32860dc19c9feed616c732bdfef7d6 --- /dev/null +++ b/9jw2ghih/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:334e103d059a965754021c13a86a5e060c0c9e272dd7a3373653c16902287d2b +size 1161166 diff --git a/9jw2ghih/eval_video/1_eval.mp4 b/9jw2ghih/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bd2116c751563627c9dc60c101351e62b71c6be --- /dev/null +++ b/9jw2ghih/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98b651e687a344add1e9f1ab00ed83622eb898a8e4c4a8ae1633bbc653019ede +size 1164220 diff --git a/9jw2ghih/eval_video/2_eval.mp4 b/9jw2ghih/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70b1935d3f98337abca59d52353bd19f163ce166 --- /dev/null +++ b/9jw2ghih/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed6142ce9bae4cdd11b43dbb86436cdc5b11c8aeb06e9c855bcba5c241db1871 +size 1160754 diff --git a/9jw2ghih/eval_video/3_eval.mp4 b/9jw2ghih/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c26027dc920e5781879662a9aff8ff94275ad056 --- /dev/null +++ b/9jw2ghih/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5145a858b26112ff00ca0e10c42c3964d24be04a764b99fa06b1ecbc727c767b +size 1162079 diff --git a/9jw2ghih/eval_video/4_eval.mp4 b/9jw2ghih/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62470e707f75de4d04434d49c31b23bfbeaf09e4 --- /dev/null +++ b/9jw2ghih/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fdfcf5c8232760435dea03aa0eb0565baaff25f323a462d14a6bb2b15688714 +size 1174471 diff --git a/9jw2ghih/tb/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 b/9jw2ghih/tb/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 new file mode 100644 index 0000000000000000000000000000000000000000..224b7adf7ec5438aecff289aa87b17dee5a5452e --- /dev/null +++ b/9jw2ghih/tb/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194787fa0a92bbcb97200c02dae4ed74e49fda5587b546fef140bf45716bcb6b +size 6539 diff --git a/9jw2ghih/wandb/debug-internal.log b/9jw2ghih/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..213304a19bc0a9f24a05e2e0fcbfdf6cecb97735 --- /dev/null +++ b/9jw2ghih/wandb/debug-internal.log @@ -0,0 +1,713 @@ +2025-01-06 20:40:56,138 INFO StreamThr :2113420 [internal.py:wandb_internal():85] W&B internal server running at pid: 2113420, started at: 2025-01-06 20:40:56.138628 +2025-01-06 20:40:56,141 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:40:56,152 INFO WriterThread:2113420 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/run-9jw2ghih.wandb +2025-01-06 20:40:56,153 DEBUG SenderThread:2113420 [sender.py:send():379] send: header +2025-01-06 20:40:56,162 DEBUG SenderThread:2113420 [sender.py:send():379] send: run +2025-01-06 20:40:56,436 INFO SenderThread:2113420 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files +2025-01-06 20:40:56,436 INFO SenderThread:2113420 [sender.py:_start_run_threads():1188] run started: 9jw2ghih with start time 1736214056.139121 +2025-01-06 20:40:56,451 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:40:56,451 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:40:56,541 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:40:56,546 DEBUG HandlerThread:2113420 [system_info.py:__init__():26] System info init +2025-01-06 20:40:56,546 DEBUG HandlerThread:2113420 [system_info.py:__init__():41] System info init done +2025-01-06 20:40:56,546 INFO HandlerThread:2113420 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:40:56,546 INFO SystemMonitor:2113420 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:40:56,547 INFO HandlerThread:2113420 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:40:56,547 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:40:56,547 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:40:56,548 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:40:56,549 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:40:56,549 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:40:56,559 DEBUG HandlerThread:2113420 [system_info.py:probe():152] Probing system +2025-01-06 20:40:56,561 DEBUG HandlerThread:2113420 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:40:56,565 DEBUG HandlerThread:2113420 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:40:56,565 DEBUG HandlerThread:2113420 [system_info.py:probe():200] Probing system done +2025-01-06 20:40:56,566 DEBUG HandlerThread:2113420 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:40:56.559836', 'startedAt': '2025-01-07T01:40:56.130850', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.883, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.986, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.08, 'min': 800.0, 'max': 4700.0}, {'current': 4493.337, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 512.135066986084}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:40:56,566 INFO HandlerThread:2113420 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:40:56,566 INFO HandlerThread:2113420 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:40:56,566 DEBUG HandlerThread:2113420 [system_info.py:_save_code():44] Saving code +2025-01-06 20:40:56,567 DEBUG HandlerThread:2113420 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:40:56,567 DEBUG HandlerThread:2113420 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:40:56,633 DEBUG HandlerThread:2113420 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:40:56,634 INFO HandlerThread:2113420 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:40:56,637 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:40:56,637 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:40:56,637 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:40:56,638 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:40:56,884 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/12bgrhpl-code/FISH/eval_robot.py +2025-01-06 20:40:56,903 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:40:56,903 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:40:56,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:40:56,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:40:56,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:56,907 INFO HandlerThread:2113420 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/tb" + save: true +} +control { + relay_id: "f26cb0b48bde449899b2ec9401c66e25" +} +_info { + stream_id: "9jw2ghih" +} + +2025-01-06 20:40:56,907 DEBUG HandlerThread:2113420 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:40:56,932 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/4zzzt778-diff.patch +2025-01-06 20:40:56,992 DEBUG SenderThread:2113420 [sender.py:send():379] send: telemetry +2025-01-06 20:40:56,992 DEBUG SenderThread:2113420 [sender.py:send():379] send: tbrecord +2025-01-06 20:40:56,993 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:40:56,993 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 with policy live +2025-01-06 20:40:57,193 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/con9qt36-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:40:57,278 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/x3f07fhk-wandb-metadata.json +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/diff.patch +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-metadata.json +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH/eval_robot.py +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH +2025-01-06 20:40:57,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:58,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:59,437 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:40:59,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:00,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:01,149 DEBUG SenderThread:2113420 [sender.py:send():379] send: config +2025-01-06 20:41:01,150 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:01,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:02,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:03,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:03,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:04,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:05,921 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:06,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:06,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:07,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:07,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:08,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:09,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:10,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:11,441 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:11,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:11,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:11,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:12,061 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:12,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:13,441 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:13,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:14,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:15,442 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:15,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:16,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:17,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:17,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:18,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:19,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:20,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:21,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:22,908 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:23,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:23,908 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:24,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:25,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:26,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:26,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:26,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:27,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:28,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:29,024 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:29,445 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml +2025-01-06 20:41:29,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:30,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:31,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:32,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:33,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:34,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:34,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:35,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:36,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:37,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:38,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:39,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:40,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:40,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:41,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:41,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:41,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:42,213 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:41:42,214 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:41:42,214 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_334e103d059a96575402.mp4 with policy now +2025-01-06 20:41:42,214 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:41:42,214 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:41:42,215 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media +2025-01-06 20:41:42,467 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/24eluwyi-media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:41:42,726 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/sytwvzmi-media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:41:42,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:43,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:43,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:44,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:45,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:46,845 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:46,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:47,451 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:47,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:48,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:49,451 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:49,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:50,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:51,452 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:51,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:51,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:52,452 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:41:52,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:53,453 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:53,914 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:54,914 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:55,454 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:55,914 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:56,550 DEBUG SystemMonitor:2113420 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:41:56,550 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:41:56,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:56,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:56,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:57,035 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:57,454 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:57,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:58,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:59,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:00,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:01,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:02,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:02,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:03,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:04,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:05,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:06,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:07,908 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:07,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:08,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:09,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:10,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:11,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:11,905 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:11,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:12,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:13,019 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:13,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:14,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:15,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:16,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:17,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:18,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:18,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:19,461 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:19,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:20,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:21,462 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:21,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:22,918 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:23,796 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:42:23,796 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 with policy now +2025-01-06 20:42:23,798 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:42:23,799 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:42:23,799 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:42:23,799 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:42:23,919 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:24,024 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/hj2z2fnq-media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:42:24,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:42:24,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:42:24,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:42:24,727 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/89fbs4p0-media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:42:24,813 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:24,921 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:25,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:25,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:26,551 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:42:26,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:26,906 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:26,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:27,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:28,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:29,465 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:29,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:30,191 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:30,921 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:31,465 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:31,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:32,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:33,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:34,294 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:42:34,294 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:42:34,295 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:42:34,467 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:42:34,467 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:42:34,707 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/vygcdxtl-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:42:34,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:35,467 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:35,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:35,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:36,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:37,468 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:37,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:38,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:39,468 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:39,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:40,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:41,838 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:41,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:41,905 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:41,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:42,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:43,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:44,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:45,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:46,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:46,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:47,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:48,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:49,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:50,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:51,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:51,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:52,925 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:53,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:54,925 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:55,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:56,551 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:42:56,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:56,905 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:56,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:57,015 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:57,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:58,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:59,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:00,927 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:01,927 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:02,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:02,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:03,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:04,934 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:05,477 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:05,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:05,997 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:43:05,998 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:43:05,998 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 with policy now +2025-01-06 20:43:05,998 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:05,998 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:05,998 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:06,219 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/dnfpm6s7-media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:43:06,477 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:06,477 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:43:06,478 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:43:06,734 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/eyq4s3hs-media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:43:06,934 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:07,478 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:07,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:07,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:08,478 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:08,934 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:09,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:10,935 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:11,479 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:11,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:11,910 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:11,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:12,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:13,480 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:13,561 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:13,936 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:14,480 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:43:14,938 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:15,129 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:15,129 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:15,129 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:15,480 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:15,481 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:15,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:16,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:17,481 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:17,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:18,582 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:18,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:19,482 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:19,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:20,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:21,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:22,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:23,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:23,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:24,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:25,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:26,552 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:43:26,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:26,906 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:26,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:27,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:28,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:28,988 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:29,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:30,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:31,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:32,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:33,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:34,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:34,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:35,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:36,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:37,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:38,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:39,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:39,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:40,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:41,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:41,906 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:41,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:42,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:43,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:44,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:45,013 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:45,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:46,813 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:43:46,813 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 with policy now +2025-01-06 20:43:46,817 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:43:46,818 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:46,818 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:46,819 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:46,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:47,033 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/yzj30ono-media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:43:47,762 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/200233rw-media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:43:47,948 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:48,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:48,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:49,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:49,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:50,463 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:50,948 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:51,494 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:51,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:52,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:53,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:54,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:55,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:55,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:43:55,781 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/1kdksc1h-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:43:55,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:56,176 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:56,177 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:56,177 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:56,177 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:56,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:56,552 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:43:56,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:56,912 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:56,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:57,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:57,954 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:58,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:59,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:00,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:01,498 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:01,533 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:01,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:02,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:03,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:04,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:05,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:06,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:06,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:07,500 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:07,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:08,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:09,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:10,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:11,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:44:11,910 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:44:11,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:12,041 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:12,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:13,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:14,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:15,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:16,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:17,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:17,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:18,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:19,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:20,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:21,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:22,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:23,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:23,956 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:24,954 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:25,955 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:26,553 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:44:26,935 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:44:26,935 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:44:26,977 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:27,507 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:27,564 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:44:27,564 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 with policy now +2025-01-06 20:44:27,567 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:44:27,567 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:27,567 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:27,567 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:27,790 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/u4iaoxdh-media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:27,958 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:28,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:28,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:28,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:44:28,791 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/e8wimxyz-media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:28,968 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:29,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:29,582 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:29,961 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:30,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:30,974 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:31,968 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:32,971 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:33,510 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:33,981 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:34,986 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:35,137 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:35,510 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:35,980 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:36,977 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:37,981 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:38,979 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:39,979 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:40,234 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:40,234 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:40,234 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:40,235 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:40,267 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:44:40,267 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:44:40,493 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media +2025-01-06 20:44:40,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table +2025-01-06 20:44:40,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval +2025-01-06 20:44:40,701 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/wzli8mcj-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:40,759 INFO wandb-upload_0:2113420 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpss903jkv +2025-01-06 20:44:40,880 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:44:40,889 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:44:40,976 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:41,183 INFO SenderThread:2113420 [sender.py:send_request_log_artifact():1525] logged artifact run-9jw2ghih-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMxNjAyMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0MjczMQ==', 'latestArtifact': None}} +2025-01-06 20:44:41,183 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:44:41,184 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 20:44:41,184 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:41,184 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,184 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,185 DEBUG SenderThread:2113420 [sender.py:send():379] send: telemetry +2025-01-06 20:44:41,185 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,188 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,188 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,189 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,189 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,189 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,189 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,190 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,190 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,190 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,190 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,190 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,191 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,191 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,191 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,191 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,191 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,192 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,192 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,192 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,192 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,193 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,193 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:44:41,467 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/gl5j253z-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval +2025-01-06 20:44:41,594 INFO wandb-upload_0:2113420 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo3wa4c97 +2025-01-06 20:44:41,740 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/lg8qv5sm-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:41,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:44:41,976 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:42,018 INFO SenderThread:2113420 [sender.py:send_request_log_artifact():1525] logged artifact run-9jw2ghih-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMxNjAzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0MjczNQ==', 'latestArtifact': None}} +2025-01-06 20:44:42,019 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:44:42,019 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json with policy now +2025-01-06 20:44:42,019 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:42,019 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:42,020 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:42,020 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:44:42,273 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/m67npqkg-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:42,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:42,976 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:43,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:43,669 DEBUG SenderThread:2113420 [sender.py:send():379] send: exit +2025-01-06 20:44:43,669 INFO SenderThread:2113420 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:send_exit():588] handling runtime: 227 +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:send_exit():594] send defer +2025-01-06 20:44:43,670 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,670 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:44:43,670 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:44:43,671 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,671 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:44:43,671 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,671 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:44:43,671 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:44:43,671 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,671 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:44:43,671 INFO HandlerThread:2113420 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:44:43,672 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:44:43,672 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:44:43,672 DEBUG SystemMonitor:2113420 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:44:43,672 DEBUG SystemMonitor:2113420 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:44:43,723 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:44:43,723 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:44:43,723 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:44:43,724 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:44:43,724 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:44:43,724 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,724 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:44:43,724 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:44:43,724 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,725 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:44:44,514 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:45,514 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:46,515 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml +2025-01-06 20:44:50,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:50,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:50,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:44:50,238 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,238 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:44:50,238 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:44:50,238 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:50,238 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:44:50,239 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:50,239 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:50,239 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:50,241 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:50,241 DEBUG SenderThread:2113420 [sender.py:send():379] send: summary +2025-01-06 20:44:50,245 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:50,245 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,245 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:44:50,245 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:44:50,246 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:50,246 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:44:50,246 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,246 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:44:50,246 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:44:50,246 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:50,246 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:50,246 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:44:50,246 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,246 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:44:50,516 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:50,670 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:51,819 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:44:51,819 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:51,820 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:51,820 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:44:51,820 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:51,820 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:44:51,820 INFO SenderThread:2113420 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:44:51,822 INFO SenderThread:2113420 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:44:51,824 INFO SenderThread:2113420 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:44:51,842 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:44:51,843 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:51,843 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:44:51,843 DEBUG SenderThread:2113420 [sender.py:send():379] send: artifact +2025-01-06 20:44:52,046 INFO SenderThread:2113420 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:44:52,046 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:52,046 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:44:52,046 INFO SenderThread:2113420 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:44:52,517 INFO SenderThread:2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:52,517 INFO SenderThread:2113420 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml config.yaml +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json wandb-summary.json +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt requirements.txt +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log output.log +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/diff.patch diff.patch +2025-01-06 20:44:52,523 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:44:52,527 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:52,527 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:44:52,527 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:44:52,671 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:52,672 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:52,713 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt +2025-01-06 20:44:52,718 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml +2025-01-06 20:44:52,730 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:52,735 INFO wandb-upload_4:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:52,829 INFO wandb-upload_3:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/il34sskj-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:53,029 INFO Thread-11 (_thread_body):2113420 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:44:53,029 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,029 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:44:53,029 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,030 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:44:53,030 INFO SenderThread:2113420 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:44:53,033 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:44:53,034 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,034 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:44:53,034 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,034 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:44:53,034 INFO SenderThread:2113420 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:44:53,234 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,234 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:44:53,234 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:44:53,235 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,235 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:44:53,235 DEBUG SenderThread:2113420 [sender.py:send():379] send: final +2025-01-06 20:44:53,235 DEBUG SenderThread:2113420 [sender.py:send():379] send: footer +2025-01-06 20:44:53,235 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,235 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:44:53,236 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:53,236 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:44:53,238 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:53,238 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:44:53,292 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:53,292 INFO MainThread:2113420 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:44:53,293 INFO MainThread:2113420 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:44:53,293 INFO MainThread:2113420 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:44:53,293 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:44:53,293 INFO HandlerThread:2113420 [handler.py:finish():882] shutting down handler +2025-01-06 20:44:54,237 INFO WriterThread:2113420 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/run-9jw2ghih.wandb +2025-01-06 20:44:54,292 INFO SenderThread:2113420 [sender.py:finish():1615] shutting down sender +2025-01-06 20:44:54,292 INFO SenderThread:2113420 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:44:54,292 INFO SenderThread:2113420 [file_pusher.py:join():175] waiting for file pusher diff --git a/9jw2ghih/wandb/debug.log b/9jw2ghih/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4f6fed444226fc43096bf7d94193a676b8875501 --- /dev/null +++ b/9jw2ghih/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Configure stats pid to 2113240 +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/settings +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug.log +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug-internal.log +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih'} +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():619] starting backend +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():623] setting up manager +2025-01-06 20:40:56,137 INFO MainThread:2113240 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:40:56,138 INFO MainThread:2113240 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:40:56,152 INFO MainThread:2113240 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:40:56,159 INFO MainThread:2113240 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:40:56,450 INFO MainThread:2113240 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:40:56,536 INFO MainThread:2113240 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:40:56,536 INFO MainThread:2113240 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:40:56,904 INFO MainThread:2113240 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:40:56,904 INFO MainThread:2113240 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:40:56,904 INFO MainThread:2113240 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:40:56,905 INFO MainThread:2113240 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:40:56,906 INFO MainThread:2113240 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:40:56,906 INFO MainThread:2113240 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/tb, True +2025-01-06 20:41:01,149 INFO MainThread:2113240 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:44:54,294 WARNING MsgRouterThr:2113240 [router.py:message_loop():77] message_loop has been closed diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH/eval_robot.py b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51003296046b81a922703c99596595e51981d208 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736214056 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/diff.patch b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..43dd1a505c5d25a547a09b1d8e6dc31eb2a6a985 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..101f9bd 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json new file mode 100644 index 0000000000000000000000000000000000000000..5271efacb3db6a0813497da4611af43307900d95 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 15, 1.0, 0.9928352235762072, 1, 0.7993280141947043, 1, 0.8249338715618673, 1, 1, 1]]} \ No newline at end of file diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d36c06c0db32860dc19c9feed616c732bdfef7d6 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:334e103d059a965754021c13a86a5e060c0c9e272dd7a3373653c16902287d2b +size 1161166 diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bd2116c751563627c9dc60c101351e62b71c6be --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98b651e687a344add1e9f1ab00ed83622eb898a8e4c4a8ae1633bbc653019ede +size 1164220 diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..70b1935d3f98337abca59d52353bd19f163ce166 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed6142ce9bae4cdd11b43dbb86436cdc5b11c8aeb06e9c855bcba5c241db1871 +size 1160754 diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c26027dc920e5781879662a9aff8ff94275ad056 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5145a858b26112ff00ca0e10c42c3964d24be04a764b99fa06b1ecbc727c767b +size 1162079 diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62470e707f75de4d04434d49c31b23bfbeaf09e4 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fdfcf5c8232760435dea03aa0eb0565baaff25f323a462d14a6bb2b15688714 +size 1174471 diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..deefb65df9be4a4fc98c1cc3df779d0f71832c0f --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736214061.149971]: resetting environment +[INFO] [1736214061.153145]: cleared current plan +[INFO] [1736214061.154048]: moving to home +[INFO] [1736214062.156811]: reached home +[INFO] [1736214062.159652]: reset action history +[INFO] [1736214064.871089]: environment reset +Starting episode 0 +[INFO] [1736214064.873658]: resetting environment +[INFO] [1736214064.877773]: cleared current plan +[INFO] [1736214064.878657]: moving to home +[INFO] [1736214065.880653]: reached home +[INFO] [1736214065.881888]: reset action history +[INFO] [1736214068.594195]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736214070.509531]: resetting environment +[INFO] [1736214070.512808]: cleared current plan +[INFO] [1736214070.514151]: moving to home +[INFO] [1736214071.516404]: reached home +[INFO] [1736214071.517886]: reset action history +[INFO] [1736214074.225755]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/episode_rosbags/episode_0_2025-01-06-20-41-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2113240.log +started roslaunch server http://158.130.50.37:36939/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2113546] +process[print_text-2]: started with pid [2113547] +process[rosbag_record-3]: started with pid [2113548] +started bagging! +[INFO] [1736214101.235106]: Storing episode... +[rosbag_record-3] killing on exit[pub_text-1] killing on exit[print_text-2] killing on exit +[INFO] [1736214101.969350]: Stored episode 1. +[INFO] [1736214101.969940]: Saving video... +[INFO] [1736214102.169563]: Video saved! +Starting episode 1 +[INFO] [1736214102.184795]: resetting environment +[INFO] [1736214102.198776]: cleared current plan +[INFO] [1736214102.199406]: moving to home +[INFO] [1736214105.801384]: reached home +[INFO] [1736214105.802046]: reset action history +[INFO] [1736214108.511234]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214111.953924]: resetting environment +[INFO] [1736214111.957892]: cleared current plan +[INFO] [1736214111.958064]: moving to home +[INFO] [1736214112.959680]: reached home +[INFO] [1736214112.960082]: reset action history +[INFO] [1736214115.666320]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/episode_rosbags/episode_1_2025-01-06-20-41-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2113240.log +started roslaunch server http://158.130.50.37:41187/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2113656] +process[print_text-5]: started with pid [2113657] +process[rosbag_record-6]: started with pid [2113658] +started bagging! +For topic gripper_width: timestamp difference is -109624817 for nearest: 1736214136633261249 - target: 1736214136742886066 at idx 249 +For topic gripper_width: timestamp difference is -107701086 for nearest: 1736214139033228613 - target: 1736214139140929699 at idx 249 +[INFO] [1736214142.601515]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736214143.399138]: Stored episode 2. +[INFO] [1736214143.400026]: Saving video... +[INFO] [1736214143.761882]: Video saved! +Starting episode 2 +[INFO] [1736214143.801197]: resetting environment +[INFO] [1736214143.812668]: cleared current plan +[INFO] [1736214143.812972]: moving to home +[INFO] [1736214147.416686]: reached home +[INFO] [1736214147.420736]: reset action history +[INFO] [1736214150.190676]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214153.880131]: resetting environment +[INFO] [1736214153.889800]: cleared current plan +[INFO] [1736214153.895359]: moving to home +[INFO] [1736214154.902323]: reached home +[INFO] [1736214154.902809]: reset action history +[INFO] [1736214157.610188]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/episode_rosbags/episode_2_2025-01-06-20-42-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2113240.log +started roslaunch server http://158.130.50.37:38827/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2113795] +process[print_text-8]: started with pid [2113796] +process[rosbag_record-9]: started with pid [2113797] +started bagging! +[INFO] [1736214184.539444]: Storing episode... +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[INFO] [1736214185.355066]: Stored episode 3. +[INFO] [1736214185.357317]: Saving video... +[INFO] [1736214185.956565]: Video saved! +Starting episode 3 +[INFO] [1736214185.998897]: resetting environment +[INFO] [1736214186.009611]: cleared current plan +[INFO] [1736214186.011299]: moving to home +[WARN] [1736214185.711492]: Plan exhausted +[WARN] [1736214185.753788]: Plan exhausted +[WARN] [1736214185.791491]: Plan exhausted +[WARN] [1736214185.833171]: Plan exhausted +[WARN] [1736214185.871769]: Plan exhausted +[WARN] [1736214185.912697]: Plan exhausted +[WARN] [1736214185.952095]: Plan exhausted +[WARN] [1736214185.992039]: Plan exhausted +[INFO] [1736214189.722587]: reached home +[INFO] [1736214189.726282]: reset action history +[INFO] [1736214192.517663]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214194.391233]: resetting environment +[INFO] [1736214194.401675]: cleared current plan +[INFO] [1736214194.402518]: moving to home +[INFO] [1736214195.406388]: reached home +[INFO] [1736214195.408185]: reset action history +[INFO] [1736214198.353342]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/episode_rosbags/episode_3_2025-01-06-20-43-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2113240.log +started roslaunch server http://158.130.50.37:39099/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2113930] +process[print_text-11]: started with pid [2113931] +process[rosbag_record-12]: started with pid [2113955] +started bagging! +[INFO] [1736214225.314990]: Storing episode... +[rosbag_record-12] killing on exit[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736214226.103366]: Stored episode 4. +[INFO] [1736214226.107769]: Saving video... +[WARN] [1736214226.455123]: Plan exhausted +[WARN] [1736214226.495080]: Plan exhausted +[WARN] [1736214226.537473]: Plan exhausted +[WARN] [1736214226.577517]: Plan exhausted +[WARN] [1736214226.615807]: Plan exhausted +[WARN] [1736214226.655519]: Plan exhausted +[WARN] [1736214226.702343]: Plan exhausted +[WARN] [1736214226.734981]: Plan exhausted +[WARN] [1736214226.774820]: Plan exhausted +[WARN] [1736214226.815143]: Plan exhausted +[WARN] [1736214226.854963]: Plan exhausted +[INFO] [1736214226.770490]: Video saved! +Starting episode 4 +[INFO] [1736214226.829506]: resetting environment +[INFO] [1736214226.857465]: cleared current plan +[INFO] [1736214226.857642]: moving to home +[INFO] [1736214230.461313]: reached home +[INFO] [1736214230.469759]: reset action history +[INFO] [1736214233.249079]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214235.347336]: resetting environment +[INFO] [1736214235.370764]: cleared current plan +[INFO] [1736214235.375339]: moving to home +[INFO] [1736214236.379398]: reached home +[INFO] [1736214236.383020]: reset action history +[INFO] [1736214239.106941]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/episode_rosbags/episode_4_2025-01-06-20-43-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2113240.log +started roslaunch server http://158.130.50.37:40467/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2114086] +process[print_text-14]: started with pid [2114087] +process[rosbag_record-15]: started with pid [2114088] +started bagging! +For topic gripper_width: timestamp difference is -122342542 for nearest: 1736214244833227202 - target: 1736214244955569744 at idx 91 +[INFO] [1736214266.236139]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736214267.037324]: Stored episode 5. +[INFO] [1736214267.055994]: Saving video... +[INFO] [1736214267.527136]: Video saved! +[INFO] [1736214267.566401]: resetting environment +[INFO] [1736214267.581483]: cleared current plan +[INFO] [1736214267.581739]: moving to home +[WARN] [1736214267.576931]: Plan exhausted +[INFO] [1736214271.289104]: reached home +[INFO] [1736214271.300719]: reset action history +[INFO] [1736214274.096233]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-metadata.json b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5e7703f0f29efed2e713ee4c12585ff5b43e7655 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:40:56.559836", + "startedAt": "2025-01-07T01:40:56.130850", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.883, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.986, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.08, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4493.337, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 512.135066986084 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d04b01fdeff188f7c7929bcbf368af53fb6f84f5 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "334e103d059a965754021c13a86a5e060c0c9e272dd7a3373653c16902287d2b", "size": 1161166, "path": "media/videos/eval/0_eval_0_334e103d059a96575402.mp4"}, "global_step": 663, "_timestamp": 1736214280.7727897, "_runtime": 224.63366866111755, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "98b651e687a344add1e9f1ab00ed83622eb898a8e4c4a8ae1633bbc653019ede", "size": 1164220, "path": "media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4"}, "eval/num_success": 5.0, "episode": 15.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "ed6142ce9bae4cdd11b43dbb86436cdc5b11c8aeb06e9c855bcba5c241db1871", "size": 1160754, "path": "media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "5145a858b26112ff00ca0e10c42c3964d24be04a764b99fa06b1ecbc727c767b", "size": 1162079, "path": "media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "9fdfcf5c8232760435dea03aa0eb0565baaff25f323a462d14a6bb2b15688714", "size": 1174471, "path": "media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://wthg50x6ip8z1s3l0hpgdgwsycb9kjrrbbttm3o6sq4za1g653dikgix30kb24ieehz9ja1hxyjccbif6zm9xwmz0vm35i149tt72xfy8o5nyqchp1y4a2dq7bsmefgz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xb88hqp58p7a5uhvsi5hia5vets4nn0j0uofbad63lpu796pij7pnuxrwcdbms2xhz29qj4aaha286vzrn2y18cd8lihcugnlxcguiadsz4f4tvpbtjmjyw0cwlwv9i7:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 15, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9928352235762072, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7993280141947043, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.8249338715618673, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6e312070ee781a67b6ed562ed805cc283e04b6daa3f3dedf9db9ab8790e00e5b", "size": 420, "artifact_path": "wandb-client-artifact://rplma3kq4ia65xgbqep8bxqxinfjq6md0goye14kjbzsvvojp9tz9vp8xbce724qnu8qd52rgb0rqoq9co69pgckb138ga0crj9eunmdauhqw96d34d8il293u1xg6ov/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://l2duhcfl921qgi9g1lcf021jrq8d91znanje4qww56luhysbz2td32dpne5gwukif7dbc2tqca9hixqh86cx0b1hqx8zqbtglee13rjfvsa7dh3adxmmldnpxztx8v7i:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 227}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9928352236747742, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7993280291557312, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8249338865280151, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug-internal.log b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..213304a19bc0a9f24a05e2e0fcbfdf6cecb97735 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug-internal.log @@ -0,0 +1,713 @@ +2025-01-06 20:40:56,138 INFO StreamThr :2113420 [internal.py:wandb_internal():85] W&B internal server running at pid: 2113420, started at: 2025-01-06 20:40:56.138628 +2025-01-06 20:40:56,141 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:40:56,152 INFO WriterThread:2113420 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/run-9jw2ghih.wandb +2025-01-06 20:40:56,153 DEBUG SenderThread:2113420 [sender.py:send():379] send: header +2025-01-06 20:40:56,162 DEBUG SenderThread:2113420 [sender.py:send():379] send: run +2025-01-06 20:40:56,436 INFO SenderThread:2113420 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files +2025-01-06 20:40:56,436 INFO SenderThread:2113420 [sender.py:_start_run_threads():1188] run started: 9jw2ghih with start time 1736214056.139121 +2025-01-06 20:40:56,451 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:40:56,451 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:40:56,541 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:40:56,546 DEBUG HandlerThread:2113420 [system_info.py:__init__():26] System info init +2025-01-06 20:40:56,546 DEBUG HandlerThread:2113420 [system_info.py:__init__():41] System info init done +2025-01-06 20:40:56,546 INFO HandlerThread:2113420 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:40:56,546 INFO SystemMonitor:2113420 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:40:56,547 INFO HandlerThread:2113420 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:40:56,547 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:40:56,547 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:40:56,548 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:40:56,549 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:40:56,549 INFO SystemMonitor:2113420 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:40:56,559 DEBUG HandlerThread:2113420 [system_info.py:probe():152] Probing system +2025-01-06 20:40:56,561 DEBUG HandlerThread:2113420 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:40:56,565 DEBUG HandlerThread:2113420 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:40:56,565 DEBUG HandlerThread:2113420 [system_info.py:probe():200] Probing system done +2025-01-06 20:40:56,566 DEBUG HandlerThread:2113420 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:40:56.559836', 'startedAt': '2025-01-07T01:40:56.130850', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.883, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.986, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.08, 'min': 800.0, 'max': 4700.0}, {'current': 4493.337, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 512.135066986084}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:40:56,566 INFO HandlerThread:2113420 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:40:56,566 INFO HandlerThread:2113420 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:40:56,566 DEBUG HandlerThread:2113420 [system_info.py:_save_code():44] Saving code +2025-01-06 20:40:56,567 DEBUG HandlerThread:2113420 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:40:56,567 DEBUG HandlerThread:2113420 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:40:56,633 DEBUG HandlerThread:2113420 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:40:56,634 INFO HandlerThread:2113420 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:40:56,637 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:40:56,637 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:40:56,637 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:40:56,638 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:40:56,884 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/12bgrhpl-code/FISH/eval_robot.py +2025-01-06 20:40:56,903 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:40:56,903 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:40:56,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:40:56,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:40:56,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:56,907 INFO HandlerThread:2113420 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/tb" + save: true +} +control { + relay_id: "f26cb0b48bde449899b2ec9401c66e25" +} +_info { + stream_id: "9jw2ghih" +} + +2025-01-06 20:40:56,907 DEBUG HandlerThread:2113420 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:40:56,932 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/4zzzt778-diff.patch +2025-01-06 20:40:56,992 DEBUG SenderThread:2113420 [sender.py:send():379] send: telemetry +2025-01-06 20:40:56,992 DEBUG SenderThread:2113420 [sender.py:send():379] send: tbrecord +2025-01-06 20:40:56,993 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:40:56,993 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 with policy live +2025-01-06 20:40:57,193 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/con9qt36-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:40:57,278 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/x3f07fhk-wandb-metadata.json +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/diff.patch +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-metadata.json +2025-01-06 20:40:57,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH/eval_robot.py +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code +2025-01-06 20:40:57,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH +2025-01-06 20:40:57,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:58,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:59,437 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:40:59,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:00,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:01,149 DEBUG SenderThread:2113420 [sender.py:send():379] send: config +2025-01-06 20:41:01,150 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:01,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:02,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:03,438 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:03,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:04,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:05,921 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:06,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:06,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:07,439 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:07,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:08,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:09,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:10,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:11,441 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:11,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:11,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:11,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:12,061 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:12,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:13,441 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:13,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:14,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:15,442 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:15,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:16,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:17,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:17,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:18,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:19,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:20,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:21,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:22,908 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:23,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:23,908 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:24,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:25,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:26,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:26,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:26,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:27,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:28,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:29,024 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:29,445 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml +2025-01-06 20:41:29,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:30,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:31,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:32,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:33,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:34,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:34,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:35,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:36,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:37,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:38,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:39,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:40,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:40,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:41,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:41,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:41,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:42,213 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:41:42,214 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:41:42,214 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_334e103d059a96575402.mp4 with policy now +2025-01-06 20:41:42,214 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:41:42,214 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:41:42,215 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:41:42,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media +2025-01-06 20:41:42,467 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/24eluwyi-media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:41:42,726 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/sytwvzmi-media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:41:42,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:43,449 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:43,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:44,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:45,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:46,845 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:46,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:47,451 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:47,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:48,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:49,451 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:49,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:50,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:51,452 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:51,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:51,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:52,452 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:41:52,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:53,453 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:53,914 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:54,914 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:55,454 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:55,914 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:56,550 DEBUG SystemMonitor:2113420 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:41:56,550 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:41:56,904 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:41:56,904 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:41:56,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:57,035 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:41:57,454 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:41:57,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:58,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:41:59,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:00,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:01,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:02,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:02,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:03,915 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:04,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:05,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:06,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:07,908 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:07,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:08,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:09,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:10,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:11,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:11,905 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:11,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:12,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:13,019 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:13,916 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:14,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:15,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:16,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:17,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:18,907 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:18,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:19,461 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:19,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:20,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:21,462 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:21,917 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:22,918 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:23,796 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:42:23,796 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 with policy now +2025-01-06 20:42:23,798 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:42:23,799 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:42:23,799 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:42:23,799 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:42:23,919 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:24,024 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/hj2z2fnq-media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:42:24,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:42:24,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:42:24,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:42:24,727 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/89fbs4p0-media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:42:24,813 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:24,921 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:25,463 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:25,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:26,551 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:42:26,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:26,906 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:26,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:27,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:28,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:29,465 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:29,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:30,191 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:30,921 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:31,465 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:31,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:32,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:33,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:34,294 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:42:34,294 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:42:34,295 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:42:34,467 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:42:34,467 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:42:34,707 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/vygcdxtl-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:42:34,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:35,467 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:35,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:35,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:36,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:37,468 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:37,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:38,922 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:39,468 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:42:39,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:40,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:41,838 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:41,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:41,905 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:41,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:42,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:43,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:44,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:45,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:46,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:46,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:47,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:48,923 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:49,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:50,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:51,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:51,924 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:52,925 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:53,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:54,925 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:55,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:56,551 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:42:56,905 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:42:56,905 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:42:56,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:57,015 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:42:57,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:58,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:42:59,926 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:00,927 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:01,927 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:02,909 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:02,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:03,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:04,934 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:05,477 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:05,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:05,997 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:43:05,998 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:43:05,998 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 with policy now +2025-01-06 20:43:05,998 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:05,998 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:05,998 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:06,219 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/dnfpm6s7-media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:43:06,477 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:06,477 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:43:06,478 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:43:06,734 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/eyq4s3hs-media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:43:06,934 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:07,478 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:07,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:07,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:08,478 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:08,934 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:09,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:10,935 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:11,479 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:11,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:11,910 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:11,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:12,933 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:13,480 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:13,561 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:13,936 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:14,480 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:43:14,938 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:15,129 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:15,129 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:15,129 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:15,480 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:15,481 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:15,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:16,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:17,481 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:17,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:18,582 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:18,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:19,482 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:19,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:20,939 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:21,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:22,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:23,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:23,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:24,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:25,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:26,552 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:43:26,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:26,906 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:26,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:27,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:28,940 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:28,988 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:29,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:30,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:31,941 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:32,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:33,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:34,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:34,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:35,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:36,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:37,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:38,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:39,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:39,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:40,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:41,906 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:41,906 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:41,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:42,942 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:43,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:44,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:45,013 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:45,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:46,813 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:43:46,813 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 with policy now +2025-01-06 20:43:46,817 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:43:46,818 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:46,818 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:46,819 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:46,943 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:47,033 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/yzj30ono-media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:43:47,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:43:47,762 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/200233rw-media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:43:47,948 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:48,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:48,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:49,493 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:49,945 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:50,463 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:50,948 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:51,494 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:51,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:52,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:53,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:54,949 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:55,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:55,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:43:55,781 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/1kdksc1h-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:43:55,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:56,176 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:43:56,177 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:43:56,177 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:43:56,177 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:43:56,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:43:56,552 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:43:56,912 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:43:56,912 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:43:56,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:57,496 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:43:57,954 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:58,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:43:59,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:00,950 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:01,498 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:01,533 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:01,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:02,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:03,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:04,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:05,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:06,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:06,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:07,500 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:07,951 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:08,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:09,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:10,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:11,910 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:44:11,910 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:44:11,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:12,041 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:12,952 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:13,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:14,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:15,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:16,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:17,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:17,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:18,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:19,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:20,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:21,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:22,953 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:23,913 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:23,956 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:24,954 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:25,955 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:26,553 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:44:26,935 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:44:26,935 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:44:26,977 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:27,507 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:27,564 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:44:27,564 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 with policy now +2025-01-06 20:44:27,567 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:44:27,567 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:27,567 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:27,567 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:27,790 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/u4iaoxdh-media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:27,958 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:28,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:28,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:28,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval +2025-01-06 20:44:28,791 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/e8wimxyz-media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:28,968 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:29,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:29,582 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:29,961 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:30,508 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:30,974 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:31,968 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:32,971 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:33,510 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:33,981 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:34,986 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:35,137 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:35,510 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:35,980 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:36,977 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:37,981 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:38,979 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:39,979 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:40,234 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:40,234 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:40,234 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:40,235 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:40,267 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:44:40,267 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:44:40,493 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:40,512 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media +2025-01-06 20:44:40,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table +2025-01-06 20:44:40,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval +2025-01-06 20:44:40,701 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/wzli8mcj-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:40,759 INFO wandb-upload_0:2113420 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpss903jkv +2025-01-06 20:44:40,880 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:44:40,889 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:44:40,976 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:41,183 INFO SenderThread:2113420 [sender.py:send_request_log_artifact():1525] logged artifact run-9jw2ghih-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMxNjAyMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0MjczMQ==', 'latestArtifact': None}} +2025-01-06 20:44:41,183 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:44:41,184 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 20:44:41,184 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:41,184 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,184 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,185 DEBUG SenderThread:2113420 [sender.py:send():379] send: telemetry +2025-01-06 20:44:41,185 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,188 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,188 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,189 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,189 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,189 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,189 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,190 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,190 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,190 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,190 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,190 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,191 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,191 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,191 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,191 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,191 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,192 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,192 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,192 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,192 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:41,193 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:41,193 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:44:41,467 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/gl5j253z-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:41,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval +2025-01-06 20:44:41,594 INFO wandb-upload_0:2113420 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo3wa4c97 +2025-01-06 20:44:41,740 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/lg8qv5sm-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:41,911 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:44:41,976 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:42,018 INFO SenderThread:2113420 [sender.py:send_request_log_artifact():1525] logged artifact run-9jw2ghih-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMxNjAzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY0MjczNQ==', 'latestArtifact': None}} +2025-01-06 20:44:42,019 DEBUG SenderThread:2113420 [sender.py:send():379] send: files +2025-01-06 20:44:42,019 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json with policy now +2025-01-06 20:44:42,019 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:42,019 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:42,020 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:42,020 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:44:42,273 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/m67npqkg-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:42,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:42,976 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:43,513 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:43,669 DEBUG SenderThread:2113420 [sender.py:send():379] send: exit +2025-01-06 20:44:43,669 INFO SenderThread:2113420 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:send_exit():588] handling runtime: 227 +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:send_exit():594] send defer +2025-01-06 20:44:43,670 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,670 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:44:43,670 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:44:43,670 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:44:43,671 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,671 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:44:43,671 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,671 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:44:43,671 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:44:43,671 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,671 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:44:43,671 INFO HandlerThread:2113420 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:44:43,672 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:44:43,672 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:44:43,672 DEBUG SystemMonitor:2113420 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:44:43,672 DEBUG SystemMonitor:2113420 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:44:43,723 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:44:43,723 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:44:43,723 INFO HandlerThread:2113420 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:44:43,724 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:44:43,724 DEBUG SenderThread:2113420 [sender.py:send():379] send: stats +2025-01-06 20:44:43,724 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,724 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:44:43,724 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:44:43,724 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:44:43,724 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:43,725 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:44:44,514 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:45,514 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:46,515 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml +2025-01-06 20:44:50,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:50,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:50,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:44:50,238 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,238 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:44:50,238 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:44:50,238 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:50,238 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:44:50,239 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:50,239 DEBUG SenderThread:2113420 [sender.py:send():379] send: history +2025-01-06 20:44:50,239 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:44:50,241 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:50,241 DEBUG SenderThread:2113420 [sender.py:send():379] send: summary +2025-01-06 20:44:50,245 INFO SenderThread:2113420 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:44:50,245 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,245 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:44:50,245 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:44:50,246 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:50,246 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:44:50,246 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,246 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:44:50,246 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:44:50,246 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:44:50,246 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:50,246 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:44:50,246 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:50,246 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:44:50,516 INFO Thread-12 :2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:50,670 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:51,819 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:44:51,819 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:51,820 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:51,820 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:44:51,820 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:51,820 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:44:51,820 INFO SenderThread:2113420 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:44:51,822 INFO SenderThread:2113420 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:44:51,824 INFO SenderThread:2113420 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:44:51,842 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:44:51,843 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:51,843 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:44:51,843 DEBUG SenderThread:2113420 [sender.py:send():379] send: artifact +2025-01-06 20:44:52,046 INFO SenderThread:2113420 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:44:52,046 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:52,046 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:44:52,046 INFO SenderThread:2113420 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:44:52,517 INFO SenderThread:2113420 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:52,517 INFO SenderThread:2113420 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml config.yaml +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json wandb-summary.json +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt requirements.txt +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log output.log +2025-01-06 20:44:52,518 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/diff.patch diff.patch +2025-01-06 20:44:52,523 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 media/videos/eval/4_eval_4_9fdfcf5c8232760435de.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 media/videos/eval/2_eval_2_ed6142ce9bae4cdd11b4.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/0_eval_0_334e103d059a96575402.mp4 media/videos/eval/0_eval_0_334e103d059a96575402.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 media/videos/eval/1_eval_1_98b651e687a344add1e9.mp4 +2025-01-06 20:44:52,526 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 media/videos/eval/3_eval_3_5145a858b26112ff00ca.mp4 +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:44:52,527 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:52,527 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:44:52,527 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:44:52,527 INFO SenderThread:2113420 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:44:52,671 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:52,672 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:52,713 INFO wandb-upload_1:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/requirements.txt +2025-01-06 20:44:52,718 INFO wandb-upload_2:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/config.yaml +2025-01-06 20:44:52,730 INFO wandb-upload_0:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/wandb-summary.json +2025-01-06 20:44:52,735 INFO wandb-upload_4:2113420 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/files/output.log +2025-01-06 20:44:52,829 INFO wandb-upload_3:2113420 [upload_job.py:push():130] Uploaded file /tmp/tmprqxi18onwandb/il34sskj-events.out.tfevents.1736214056.leonmkim-ROG-Strix-G15CS-G15CS.2113240.0 +2025-01-06 20:44:53,029 INFO Thread-11 (_thread_body):2113420 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:44:53,029 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,029 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:44:53,029 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,030 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:44:53,030 INFO SenderThread:2113420 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:44:53,033 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:44:53,034 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,034 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:44:53,034 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,034 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:44:53,034 INFO SenderThread:2113420 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:44:53,234 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,234 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:44:53,234 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:44:53,234 INFO SenderThread:2113420 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:44:53,235 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:44:53,235 INFO HandlerThread:2113420 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:44:53,235 DEBUG SenderThread:2113420 [sender.py:send():379] send: final +2025-01-06 20:44:53,235 DEBUG SenderThread:2113420 [sender.py:send():379] send: footer +2025-01-06 20:44:53,235 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: defer +2025-01-06 20:44:53,235 INFO SenderThread:2113420 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:44:53,236 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:53,236 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:44:53,237 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:44:53,238 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:44:53,238 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:44:53,292 DEBUG SenderThread:2113420 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:44:53,292 INFO MainThread:2113420 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:44:53,293 INFO MainThread:2113420 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:44:53,293 INFO MainThread:2113420 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:44:53,293 DEBUG HandlerThread:2113420 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:44:53,293 INFO HandlerThread:2113420 [handler.py:finish():882] shutting down handler +2025-01-06 20:44:54,237 INFO WriterThread:2113420 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/run-9jw2ghih.wandb +2025-01-06 20:44:54,292 INFO SenderThread:2113420 [sender.py:finish():1615] shutting down sender +2025-01-06 20:44:54,292 INFO SenderThread:2113420 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:44:54,292 INFO SenderThread:2113420 [file_pusher.py:join():175] waiting for file pusher diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug.log b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4f6fed444226fc43096bf7d94193a676b8875501 --- /dev/null +++ b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Configure stats pid to 2113240 +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/settings +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:40:56,132 INFO MainThread:2113240 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug.log +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/logs/debug-internal.log +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih'} +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():619] starting backend +2025-01-06 20:40:56,133 INFO MainThread:2113240 [wandb_init.py:init():623] setting up manager +2025-01-06 20:40:56,137 INFO MainThread:2113240 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:40:56,138 INFO MainThread:2113240 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:40:56,152 INFO MainThread:2113240 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:40:56,159 INFO MainThread:2113240 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:40:56,450 INFO MainThread:2113240 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:40:56,536 INFO MainThread:2113240 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:40:56,536 INFO MainThread:2113240 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:40:56,904 INFO MainThread:2113240 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:40:56,904 INFO MainThread:2113240 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:40:56,904 INFO MainThread:2113240 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:40:56,905 INFO MainThread:2113240 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:40:56,906 INFO MainThread:2113240 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:40:56,906 INFO MainThread:2113240 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9jw2ghih/tb, True +2025-01-06 20:41:01,149 INFO MainThread:2113240 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:44:54,294 WARNING MsgRouterThr:2113240 [router.py:message_loop():77] message_loop has been closed diff --git a/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/run-9jw2ghih.wandb b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/run-9jw2ghih.wandb new file mode 100644 index 0000000000000000000000000000000000000000..565656a76966bde93d20c1f9486d3c40bbec74ac Binary files /dev/null and b/9jw2ghih/wandb/run-20250106_204056-9jw2ghih/run-9jw2ghih.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32bdde69c2c48661c2874b5ab959ab25436cbcf5 --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 1 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1009_1' +wandb_notes: 1009_1_restarted_3 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/nyyc47dd/.hydra/config.yaml b/nyyc47dd/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/nyyc47dd/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nyyc47dd/.hydra/hydra.yaml b/nyyc47dd/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab26c38b6beb3aeab6d84f5e0ac82dcf445e81a1 --- /dev/null +++ b/nyyc47dd/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nyyc47dd/.hydra/overrides.yaml b/nyyc47dd/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nyyc47dd/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nyyc47dd/episode_rosbags/aligned_depth_to_color_K.npy b/nyyc47dd/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nyyc47dd/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nyyc47dd/episode_rosbags/cam_tf_world.npy b/nyyc47dd/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nyyc47dd/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nyyc47dd/episode_rosbags/color_K.npy b/nyyc47dd/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nyyc47dd/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nyyc47dd/episode_rosbags/depth_K.npy b/nyyc47dd/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nyyc47dd/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nyyc47dd/eval_robot.log b/nyyc47dd/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..599719a81aa9500f3d741ec588f1632af756e571 --- /dev/null +++ b/nyyc47dd/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:09:13,459][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:09:13,462][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/nyyc47dd/eval_video/0_eval.mp4 b/nyyc47dd/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..993717338bd399e90828d8b4b54c0bdc91d17aaf --- /dev/null +++ b/nyyc47dd/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e0ba5727bcc50d309165e06b49549384e2ee7210bf63533dbaa463bdd33b8a +size 1125601 diff --git a/nyyc47dd/eval_video/1_eval.mp4 b/nyyc47dd/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a560bf7c70be98092e6e95d9f80fdca2709edfb2 --- /dev/null +++ b/nyyc47dd/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5e6d8239c866ea812fc8ce39286ee6c1f251de3d8cd4ccdb7b397f7412a9f26 +size 1127370 diff --git a/nyyc47dd/eval_video/2_eval.mp4 b/nyyc47dd/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa849732f4b7300cce6d9d40ef4506cf5d3c9387 --- /dev/null +++ b/nyyc47dd/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83fca2ec84021031b7bd0587d22d47a88372a0190dd8896cb21cd0c1f02ee8f5 +size 1127492 diff --git a/nyyc47dd/eval_video/3_eval.mp4 b/nyyc47dd/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1504d6da4a201bf7e8cd0e9648f028d91157b97a --- /dev/null +++ b/nyyc47dd/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:347a0bc393f70cbcbd15d1c93bc6c90e1f3a3ba88964cc9dcd82a70e6e2cd70e +size 1129420 diff --git a/nyyc47dd/eval_video/4_eval.mp4 b/nyyc47dd/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f21f36ebc50dd4150e007077d2a0673eba5ebed0 --- /dev/null +++ b/nyyc47dd/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a6b1bcbc82812416e36580441cd8b10a0498c654495bfc38eb6ad2076affbc4 +size 1124618 diff --git a/nyyc47dd/tb/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 b/nyyc47dd/tb/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 new file mode 100644 index 0000000000000000000000000000000000000000..fb06de42d41d5ddbe98f910e76c18a961504d485 --- /dev/null +++ b/nyyc47dd/tb/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15cd749068ecd97e93b25ce494185400e9ed9a17bb9690789e829f0f89847b18 +size 6539 diff --git a/nyyc47dd/wandb/debug-internal.log b/nyyc47dd/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c450acc2d9f959787ee2eb5b248fdf085c49075f --- /dev/null +++ b/nyyc47dd/wandb/debug-internal.log @@ -0,0 +1,760 @@ +2025-01-07 17:09:19,145 INFO StreamThr :2195200 [internal.py:wandb_internal():85] W&B internal server running at pid: 2195200, started at: 2025-01-07 17:09:19.144858 +2025-01-07 17:09:19,147 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:09:19,158 INFO WriterThread:2195200 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/run-nyyc47dd.wandb +2025-01-07 17:09:19,159 DEBUG SenderThread:2195200 [sender.py:send():379] send: header +2025-01-07 17:09:19,166 DEBUG SenderThread:2195200 [sender.py:send():379] send: run +2025-01-07 17:09:19,401 INFO SenderThread:2195200 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files +2025-01-07 17:09:19,401 INFO SenderThread:2195200 [sender.py:_start_run_threads():1188] run started: nyyc47dd with start time 1736287759.145573 +2025-01-07 17:09:19,417 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:09:19,417 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:09:19,463 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:09:19,468 DEBUG HandlerThread:2195200 [system_info.py:__init__():26] System info init +2025-01-07 17:09:19,468 DEBUG HandlerThread:2195200 [system_info.py:__init__():41] System info init done +2025-01-07 17:09:19,468 INFO HandlerThread:2195200 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:09:19,468 INFO SystemMonitor:2195200 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:09:19,468 INFO HandlerThread:2195200 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:09:19,469 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:09:19,469 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:09:19,470 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:09:19,470 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:09:19,471 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:09:19,483 DEBUG HandlerThread:2195200 [system_info.py:probe():152] Probing system +2025-01-07 17:09:19,485 DEBUG HandlerThread:2195200 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:09:19,489 DEBUG HandlerThread:2195200 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:09:19,489 DEBUG HandlerThread:2195200 [system_info.py:probe():200] Probing system done +2025-01-07 17:09:19,489 DEBUG HandlerThread:2195200 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:09:19.483632', 'startedAt': '2025-01-07T22:09:19.131521', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.2671250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.973, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 736.9906234741211}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:09:19,489 INFO HandlerThread:2195200 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:09:19,489 INFO HandlerThread:2195200 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:09:19,490 DEBUG HandlerThread:2195200 [system_info.py:_save_code():44] Saving code +2025-01-07 17:09:19,495 DEBUG HandlerThread:2195200 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:09:19,495 DEBUG HandlerThread:2195200 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:09:19,558 DEBUG HandlerThread:2195200 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:09:19,558 INFO HandlerThread:2195200 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:09:19,561 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:09:19,561 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:09:19,562 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:09:19,562 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:09:19,802 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/ejfpsqxu-wandb-metadata.json +2025-01-07 17:09:19,804 INFO wandb-upload_1:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/86xoq2il-code/FISH/eval_robot.py +2025-01-07 17:09:19,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:09:19,815 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:09:19,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:09:19,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:09:19,818 INFO HandlerThread:2195200 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/tb" + save: true +} +control { + relay_id: "3d3bf609a5d34bbd83bc366daf72af5e" +} +_info { + stream_id: "nyyc47dd" +} + +2025-01-07 17:09:19,818 DEBUG HandlerThread:2195200 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:09:19,836 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/769jxegn-diff.patch +2025-01-07 17:09:19,867 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:19,970 DEBUG SenderThread:2195200 [sender.py:send():379] send: telemetry +2025-01-07 17:09:19,970 DEBUG SenderThread:2195200 [sender.py:send():379] send: tbrecord +2025-01-07 17:09:19,970 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:09:19,970 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 with policy live +2025-01-07 17:09:20,149 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/84npqobk-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:09:20,402 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH/eval_robot.py +2025-01-07 17:09:20,402 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/diff.patch +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-metadata.json +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH +2025-01-07 17:09:20,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:21,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:22,402 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:22,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:23,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:24,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:24,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:25,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:26,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:27,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:28,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:29,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:29,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:30,537 DEBUG SenderThread:2195200 [sender.py:send():379] send: config +2025-01-07 17:09:30,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:31,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:32,404 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:32,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:33,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:34,405 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:34,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:09:34,815 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:09:34,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:34,971 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:35,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:36,406 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:36,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:37,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:38,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:39,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:40,407 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:40,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:40,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:41,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:42,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:43,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:44,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:45,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:46,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:46,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:47,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:48,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:49,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:09:49,815 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:09:49,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:50,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:51,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:51,959 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:52,410 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml +2025-01-07 17:09:52,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:53,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:54,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:55,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:56,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:57,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:57,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:58,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:59,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:00,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:01,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:02,413 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:02,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:02,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:03,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:04,413 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:04,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:04,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:04,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:05,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:06,414 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:06,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:07,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:07,893 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:08,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:09,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:10,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:11,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:12,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:13,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:13,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:14,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:15,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:16,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:17,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:18,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:19,192 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:19,471 DEBUG SystemMonitor:2195200 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:10:19,472 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:10:19,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:19,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:19,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:20,418 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:20,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:21,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:22,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:23,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:24,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:24,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:25,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:26,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:27,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:28,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:29,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:29,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:30,828 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:31,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:32,567 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:10:32,567 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:10:32,567 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 with policy now +2025-01-07 17:10:32,569 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:10:32,569 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:10:32,569 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:10:32,788 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/2jrwafv1-media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:10:32,828 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:33,421 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:10:33,421 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:10:33,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media +2025-01-07 17:10:33,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:10:33,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos +2025-01-07 17:10:33,681 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/ydgcet1v-media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:10:33,828 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:34,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:34,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:34,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:34,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:34,982 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:35,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:36,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:37,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:38,423 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:38,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:39,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:40,424 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:40,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:40,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:41,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:42,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:43,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:44,425 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:10:44,681 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/e928zjcx-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:10:44,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:45,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:45,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:46,426 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:46,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:47,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:48,427 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:48,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:49,472 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:10:49,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:49,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:49,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:50,427 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:50,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:50,884 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:51,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:52,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:53,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:54,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:55,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:56,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:56,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:57,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:58,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:59,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:00,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:01,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:01,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:02,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:03,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:04,432 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:04,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:04,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:04,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:05,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:06,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:06,956 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:07,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:08,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:09,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:10,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:11,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:12,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:12,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:13,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:14,836 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:15,572 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:11:15,572 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 with policy now +2025-01-07 17:11:15,575 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:11:15,576 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:11:15,576 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:11:15,577 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:11:15,829 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/asbwh4g7-media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:11:15,836 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:11:16,711 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/wkaw6lqh-media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:11:16,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:17,844 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:18,596 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:18,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:19,473 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:11:19,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:19,823 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:19,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:20,437 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:20,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:21,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:22,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:23,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:23,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:24,439 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:24,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:25,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:26,250 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:11:26,251 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:11:26,251 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:11:26,439 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:11:26,440 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:11:26,440 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:26,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:27,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:28,440 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:28,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:29,247 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:29,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:30,441 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:30,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:31,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:32,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:33,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:34,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:34,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:34,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:34,932 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:35,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:36,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:37,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:38,444 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:38,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:39,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:40,822 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:40,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:41,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:42,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:43,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:44,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:45,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:45,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:46,446 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:46,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:47,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:48,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:49,473 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:11:49,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:49,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:49,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:50,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:50,896 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:51,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:52,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:53,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:54,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:55,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:56,202 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:56,844 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:57,643 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:11:57,644 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 with policy now +2025-01-07 17:11:57,649 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:11:57,650 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:11:57,650 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:11:57,650 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:11:57,849 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:57,867 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/n5jqxyys-media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:11:58,450 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:11:58,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:58,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:11:58,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:11:58,712 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/86nqclja-media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:11:58,849 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:59,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:59,846 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:00,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:00,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:01,229 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:01,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:02,452 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:02,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:03,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:04,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:04,828 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:04,849 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:05,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:06,453 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:06,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:06,964 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:07,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:08,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:09,850 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:10,072 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:12:10,072 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:12:10,073 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:12:10,455 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:12:10,455 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:12:10,727 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/g7tkup4a-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:12:10,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:11,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:12,455 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:12,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:12,853 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:13,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:14,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:15,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:16,457 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:16,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:17,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:18,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:18,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:19,473 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:12:19,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:19,818 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:19,865 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:20,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:21,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:22,855 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:23,855 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:23,906 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:24,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:25,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:26,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:27,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:28,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:29,679 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:29,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:30,462 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:30,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:31,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:32,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:33,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:34,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:34,818 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:34,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:34,961 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:35,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:36,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:37,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:38,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:39,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:40,676 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:40,860 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:41,743 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:12:41,743 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 with policy now +2025-01-07 17:12:41,750 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:12:41,750 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:12:41,750 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:12:41,751 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:12:41,860 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:41,976 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/uvvc2mgp-media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:12:42,741 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/31gn8yf8-media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:12:42,865 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:43,866 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:44,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:44,867 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:45,864 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:46,345 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:46,467 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:46,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:47,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:48,870 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:49,474 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:12:49,822 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:49,823 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:49,871 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:50,469 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:50,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:51,122 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:12:51,122 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:12:51,122 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:12:51,469 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:12:51,470 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:12:51,782 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:51,872 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:52,470 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:52,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:53,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:54,470 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:54,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:55,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:56,471 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:56,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:56,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:57,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:58,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:59,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:00,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:01,831 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:01,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:02,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:03,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:04,474 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:04,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:13:04,818 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:13:04,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:05,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:06,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:06,974 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:07,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:08,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:09,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:10,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:11,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:12,477 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:12,561 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:12,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:13,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:14,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:15,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:16,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:17,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:17,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:18,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:19,474 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:13:19,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:13:19,819 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:13:19,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:20,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:21,879 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:22,480 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:22,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:22,887 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:22,953 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:13:22,954 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:13:22,954 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 with policy now +2025-01-07 17:13:22,954 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:22,954 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:22,954 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:23,176 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/hlgu2p5z-media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:13:23,744 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/vwtjdzut-media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:23,881 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:24,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:24,893 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:25,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:25,887 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:26,886 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:27,893 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:28,482 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:28,505 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:28,888 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:29,892 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:30,483 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:30,903 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:31,902 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:32,901 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:33,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:33,896 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:34,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:13:34,827 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:13:34,898 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:35,485 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:35,614 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:13:35,614 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:13:35,717 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/734z8zcb-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:35,809 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:13:35,896 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:36,153 INFO wandb-upload_0:2195200 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpntcl4qjp +2025-01-07 17:13:36,181 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:13:36,187 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:13:36,485 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:36,485 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table +2025-01-07 17:13:36,537 INFO SenderThread:2195200 [sender.py:send_request_log_artifact():1525] logged artifact run-nyyc47dd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NTcwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5MTkyNg==', 'latestArtifact': None}} +2025-01-07 17:13:36,537 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:13:36,537 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 17:13:36,537 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:36,537 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,538 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,538 DEBUG SenderThread:2195200 [sender.py:send():379] send: telemetry +2025-01-07 17:13:36,538 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,538 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,538 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,539 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,539 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,539 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,539 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,540 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,540 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,540 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,540 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,541 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,541 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,541 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,541 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,542 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,542 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,542 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,542 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,543 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,543 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,543 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,543 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:36,543 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,543 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,544 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:13:36,708 INFO wandb-upload_1:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/bkk08nie-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:36,750 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/zftzjo7a-media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:36,934 INFO wandb-upload_1:2195200 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp27n0xk_n +2025-01-07 17:13:36,965 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/1eoim2ae-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:37,347 INFO SenderThread:2195200 [sender.py:send_request_log_artifact():1525] logged artifact run-nyyc47dd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NTc1Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5MTkyOQ==', 'latestArtifact': None}} +2025-01-07 17:13:37,347 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:13:37,348 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json with policy now +2025-01-07 17:13:37,348 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:37,348 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:37,348 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:37,349 DEBUG SenderThread:2195200 [sender.py:send():379] send: exit +2025-01-07 17:13:37,349 INFO SenderThread:2195200 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:13:37,349 INFO SenderThread:2195200 [sender.py:send_exit():588] handling runtime: 257 +2025-01-07 17:13:37,352 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:send_exit():594] send defer +2025-01-07 17:13:37,353 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,353 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:13:37,353 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:13:37,353 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,353 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:13:37,353 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:13:37,354 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:13:37,354 DEBUG SystemMonitor:2195200 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:13:37,354 DEBUG SystemMonitor:2195200 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:13:37,408 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:13:37,408 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:13:37,409 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:13:37,409 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,409 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:13:37,409 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:13:37,409 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:13:37,409 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,409 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:13:37,410 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,410 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:13:37,410 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:13:37,410 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,410 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:13:37,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:37,592 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/b6dnw9qk-media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:38,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:40,487 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml +2025-01-07 17:13:44,181 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:44,182 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:44,182 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,183 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:13:44,183 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:44,183 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:44,184 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:13:44,185 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:44,185 DEBUG SenderThread:2195200 [sender.py:send():379] send: summary +2025-01-07 17:13:44,189 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:44,190 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:13:44,190 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:44,190 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:13:44,190 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:13:44,190 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:44,190 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:44,190 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:13:44,191 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,191 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:13:44,489 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:44,514 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:45,455 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:13:45,456 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:45,456 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:45,456 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:13:45,456 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:45,456 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:13:45,456 INFO SenderThread:2195200 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:13:45,458 INFO SenderThread:2195200 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:13:45,460 INFO SenderThread:2195200 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:13:45,471 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:13:45,471 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:45,471 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:13:45,471 DEBUG SenderThread:2195200 [sender.py:send():379] send: artifact +2025-01-07 17:13:45,489 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:45,515 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:45,934 INFO wandb-upload_1:2195200 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmprtj2_75z +2025-01-07 17:13:45,936 INFO wandb-upload_2:2195200 [upload_job.py:push():85] Skipped uploading /tmp/tmpktrvvqs7/wandb-job.json +2025-01-07 17:13:46,045 INFO wandb-upload_0:2195200 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp3877ygx +2025-01-07 17:13:46,645 INFO SenderThread:2195200 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'versionIndex': 434}}} +2025-01-07 17:13:46,645 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:46,645 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:13:46,645 INFO SenderThread:2195200 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:13:47,490 INFO SenderThread:2195200 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files +2025-01-07 17:13:47,490 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml config.yaml +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json wandb-summary.json +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt requirements.txt +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:47,495 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log output.log +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/diff.patch diff.patch +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:13:47,499 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:13:47,500 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:47,500 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:47,500 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:13:47,500 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:13:47,516 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:47,516 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:47,667 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:47,671 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt +2025-01-07 17:13:47,693 INFO wandb-upload_4:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:47,705 INFO wandb-upload_1:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml +2025-01-07 17:13:47,765 INFO wandb-upload_3:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/gnwptkbk-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:47,966 INFO Thread-11 (_thread_body):2195200 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:13:47,966 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:47,966 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:13:47,966 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:47,966 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:13:47,966 INFO SenderThread:2195200 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:13:47,968 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:13:47,968 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:47,968 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:13:47,968 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:47,968 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:13:47,968 INFO SenderThread:2195200 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:13:48,142 INFO SenderThread:2195200 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:13:48,142 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:13:48,142 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:48,142 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:13:48,142 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:48,142 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:13:48,143 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:13:48,143 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:48,143 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:13:48,143 DEBUG SenderThread:2195200 [sender.py:send():379] send: final +2025-01-07 17:13:48,143 DEBUG SenderThread:2195200 [sender.py:send():379] send: footer +2025-01-07 17:13:48,143 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:48,143 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:13:48,144 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:48,144 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:13:48,144 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:48,144 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:48,145 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:13:48,145 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:13:48,147 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:13:48,147 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:48,194 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:48,194 INFO MainThread:2195200 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:13:48,195 INFO MainThread:2195200 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:13:48,195 INFO MainThread:2195200 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:13:48,195 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:13:48,195 INFO HandlerThread:2195200 [handler.py:finish():882] shutting down handler +2025-01-07 17:13:49,146 INFO WriterThread:2195200 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/run-nyyc47dd.wandb +2025-01-07 17:13:49,194 INFO SenderThread:2195200 [sender.py:finish():1615] shutting down sender +2025-01-07 17:13:49,194 INFO SenderThread:2195200 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:13:49,194 INFO SenderThread:2195200 [file_pusher.py:join():175] waiting for file pusher diff --git a/nyyc47dd/wandb/debug.log b/nyyc47dd/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..eea2d805bdf799fff266fb7e08df42f74f4220f5 --- /dev/null +++ b/nyyc47dd/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Configure stats pid to 2195027 +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/settings +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug.log +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug-internal.log +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd'} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:init():619] starting backend +2025-01-07 17:09:19,139 INFO MainThread:2195027 [wandb_init.py:init():623] setting up manager +2025-01-07 17:09:19,144 INFO MainThread:2195027 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:09:19,145 INFO MainThread:2195027 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:09:19,158 INFO MainThread:2195027 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:09:19,164 INFO MainThread:2195027 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:09:19,416 INFO MainThread:2195027 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:09:19,458 INFO MainThread:2195027 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:09:19,458 INFO MainThread:2195027 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:09:19,817 INFO MainThread:2195027 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:09:19,817 INFO MainThread:2195027 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/tb, True +2025-01-07 17:09:30,536 INFO MainThread:2195027 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:13:49,196 WARNING MsgRouterThr:2195027 [router.py:message_loop():77] message_loop has been closed diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH/eval_robot.py b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af02abd4b07643350e5cbda642e1fb6514d4c7d0 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736287759 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/diff.patch b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c0161c26d9b20c7c85242592d6833c2df9767742 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/diff.patch @@ -0,0 +1,20 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..aa5b9b9 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,13 +134,13 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' +-# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..3c8be9b673a9235fc7e2825bc066d4aa93afd980 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[35, 35, 1.0, 0.9969230591256979, 1, 0.9084702878373503, 1, 0.9208303614609197, 1, 1, 1]]} \ No newline at end of file diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..993717338bd399e90828d8b4b54c0bdc91d17aaf --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e0ba5727bcc50d309165e06b49549384e2ee7210bf63533dbaa463bdd33b8a +size 1125601 diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a560bf7c70be98092e6e95d9f80fdca2709edfb2 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5e6d8239c866ea812fc8ce39286ee6c1f251de3d8cd4ccdb7b397f7412a9f26 +size 1127370 diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa849732f4b7300cce6d9d40ef4506cf5d3c9387 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83fca2ec84021031b7bd0587d22d47a88372a0190dd8896cb21cd0c1f02ee8f5 +size 1127492 diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1504d6da4a201bf7e8cd0e9648f028d91157b97a --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:347a0bc393f70cbcbd15d1c93bc6c90e1f3a3ba88964cc9dcd82a70e6e2cd70e +size 1129420 diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f21f36ebc50dd4150e007077d2a0673eba5ebed0 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a6b1bcbc82812416e36580441cd8b10a0498c654495bfc38eb6ad2076affbc4 +size 1124618 diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..35ab02ceecace850fd92a928e43547b1ba382f16 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736287770.537879]: resetting environment +[INFO] [1736287770.540601]: cleared current plan +[INFO] [1736287770.541376]: moving to home +[INFO] [1736287771.543813]: reached home +[INFO] [1736287771.545109]: reset action history +[INFO] [1736287774.254084]: environment reset +Starting episode 0 +[INFO] [1736287774.257453]: resetting environment +[INFO] [1736287774.262526]: cleared current plan +[INFO] [1736287774.263424]: moving to home +[INFO] [1736287775.265266]: reached home +[INFO] [1736287775.267076]: reset action history +[INFO] [1736287777.974896]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736287800.824124]: resetting environment +[INFO] [1736287800.827190]: cleared current plan +[INFO] [1736287800.828166]: moving to home +[INFO] [1736287801.830627]: reached home +[INFO] [1736287801.832121]: reset action history +[INFO] [1736287804.537924]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/episode_rosbags/episode_0_2025-01-07-17-10-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2195027.log +started roslaunch server http://158.130.50.37:41805/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2195308] +process[print_text-2]: started with pid [2195332] +process[rosbag_record-3]: started with pid [2195333] +started bagging! +For topic gripper_width: timestamp difference is 100691337 for nearest: 1736287819178871889 - target: 1736287819078180552 at idx 201 +[INFO] [1736287831.563522]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736287832.368894]: Stored episode 1. +[INFO] [1736287832.369176]: Saving video... +[INFO] [1736287832.551504]: Video saved! +Starting episode 1 +[INFO] [1736287832.568751]: resetting environment +[INFO] [1736287832.572695]: cleared current plan +[INFO] [1736287832.572934]: moving to home +[INFO] [1736287836.076814]: reached home +[INFO] [1736287836.079212]: reset action history +[INFO] [1736287838.787766]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287843.723970]: resetting environment +[INFO] [1736287843.726096]: cleared current plan +[INFO] [1736287843.726245]: moving to home +[INFO] [1736287844.727670]: reached home +[INFO] [1736287844.728081]: reset action history +[INFO] [1736287847.439158]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/episode_rosbags/episode_1_2025-01-07-17-10-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2195027.log +started roslaunch server http://158.130.50.37:45239/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2195448] +process[print_text-5]: started with pid [2195449] +process[rosbag_record-6]: started with pid [2195450] +started bagging! +For topic gripper_width: timestamp difference is -119649984 for nearest: 1736287863178876065 - target: 1736287863298526049 at idx 229 +[INFO] [1736287874.386373]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736287875.151733]: Stored episode 2. +[INFO] [1736287875.152060]: Saving video... +[INFO] [1736287875.538600]: Video saved! +Starting episode 2 +[INFO] [1736287875.577977]: resetting environment +[INFO] [1736287875.594368]: cleared current plan +[INFO] [1736287875.595694]: moving to home +[INFO] [1736287879.101000]: reached home +[INFO] [1736287879.103280]: reset action history +[INFO] [1736287881.822776]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287885.528723]: resetting environment +[INFO] [1736287885.537289]: cleared current plan +[INFO] [1736287885.537525]: moving to home +[INFO] [1736287886.539123]: reached home +[INFO] [1736287886.540401]: reset action history +[INFO] [1736287889.246724]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/episode_rosbags/episode_2_2025-01-07-17-11-29.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2195027.log +started roslaunch server http://158.130.50.37:41305/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2195582] +process[print_text-8]: started with pid [2195583] +process[rosbag_record-9]: started with pid [2195584] +started bagging! +For topic gripper_width: timestamp difference is 102999103 for nearest: 1736287895845512521 - target: 1736287895742513418 at idx 88 +For topic gripper_width: timestamp difference is 109393953 for nearest: 1736287903878845571 - target: 1736287903769451618 at idx 194 +[INFO] [1736287916.201642]: Storing episode... +[print_text-8] killing on exit +[pub_text-7] killing on exit[rosbag_record-9] killing on exit +[INFO] [1736287916.990073]: Stored episode 3. +[INFO] [1736287917.002086]: Saving video... +[INFO] [1736287917.609097]: Video saved! +[WARN] [1736287917.348057]: Plan exhausted +[WARN] [1736287917.387564]: Plan exhausted +[WARN] [1736287917.428371]: Plan exhausted +[WARN] [1736287917.467734]: Plan exhausted +[WARN] [1736287917.506791]: Plan exhausted +[WARN] [1736287917.546964]: Plan exhausted +[WARN] [1736287917.588448]: Plan exhausted +[WARN] [1736287917.627448]: Plan exhausted +[WARN] [1736287917.666850]: Plan exhausted +Starting episode 3 +[INFO] [1736287917.653276]: resetting environment +[INFO] [1736287917.669806]: cleared current plan +[INFO] [1736287917.672462]: moving to home +[INFO] [1736287921.186422]: reached home +[INFO] [1736287921.193546]: reset action history +[INFO] [1736287923.920951]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287929.781532]: resetting environment +[INFO] [1736287929.786869]: cleared current plan +[INFO] [1736287929.787151]: moving to home +[INFO] [1736287930.788722]: reached home +[INFO] [1736287930.789253]: reset action history +[INFO] [1736287933.674345]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/episode_rosbags/episode_3_2025-01-07-17-12-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2195027.log +started roslaunch server http://158.130.50.37:34307/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2195721] +process[print_text-11]: started with pid [2195722] +process[rosbag_record-12]: started with pid [2195746] +started bagging! +For topic gripper_width: timestamp difference is 113115020 for nearest: 1736287949678851075 - target: 1736287949565736055 at idx 218 +[INFO] [1736287960.676171]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736287961.428783]: Stored episode 4. +[INFO] [1736287961.432615]: Saving video... +[INFO] [1736287961.718905]: Video saved! +Starting episode 4 +[INFO] [1736287961.753488]: resetting environment +[INFO] [1736287961.780735]: cleared current plan +[INFO] [1736287961.782734]: moving to home +[INFO] [1736287965.295077]: reached home +[INFO] [1736287965.299811]: reset action history +[INFO] [1736287968.088496]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736287970.755445]: resetting environment +[INFO] [1736287970.773560]: cleared current plan +[INFO] [1736287970.775536]: moving to home +[INFO] [1736287971.779253]: reached home +[INFO] [1736287971.787490]: reset action history +[INFO] [1736287974.506580]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/episode_rosbags/episode_4_2025-01-07-17-12-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2195027.log +started roslaunch server http://158.130.50.37:43555/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2195855] +process[print_text-14]: started with pid [2195856] +process[rosbag_record-15]: started with pid [2195880] +started bagging! +For topic gripper_width: timestamp difference is -126849116 for nearest: 1736287982478847324 - target: 1736287982605696440 at idx 95 +For topic gripper_width: timestamp difference is -119675024 for nearest: 1736287991278844730 - target: 1736287991398519754 at idx 199 +[INFO] [1736288001.496287]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[WARN] [1736288002.828835]: Plan exhausted +[WARN] [1736288002.867638]: Plan exhausted +[WARN] [1736288002.907192]: Plan exhausted +[INFO] [1736288002.281412]: Stored episode 5. +[INFO] [1736288002.295802]: Saving video... +[INFO] [1736288002.856876]: Video saved! +[INFO] [1736288002.923167]: resetting environment +[INFO] [1736288002.948398]: cleared current plan +[INFO] [1736288002.950855]: moving to home +[WARN] [1736288002.946790]: Plan exhausted +[INFO] [1736288006.460631]: reached home +[INFO] [1736288006.469921]: reset action history +[INFO] [1736288009.249470]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-metadata.json b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a939f9ebee58dfa425cb8dd431901e43bfc7dcca --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:09:19.483632", + "startedAt": "2025-01-07T22:09:19.131521", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.2671250000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.973, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 736.9906234741211 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dc668afd0c405ca1c977b7ee03ff4f7f4e140e01 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "a1e0ba5727bcc50d309165e06b49549384e2ee7210bf63533dbaa463bdd33b8a", "size": 1125601, "path": "media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4"}, "global_step": 664, "_timestamp": 1736288016.095365, "_runtime": 256.94979214668274, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "d5e6d8239c866ea812fc8ce39286ee6c1f251de3d8cd4ccdb7b397f7412a9f26", "size": 1127370, "path": "media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4"}, "eval/num_success": 5.0, "episode": 35.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "83fca2ec84021031b7bd0587d22d47a88372a0190dd8896cb21cd0c1f02ee8f5", "size": 1127492, "path": "media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "347a0bc393f70cbcbd15d1c93bc6c90e1f3a3ba88964cc9dcd82a70e6e2cd70e", "size": 1129420, "path": "media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "1a6b1bcbc82812416e36580441cd8b10a0498c654495bfc38eb6ad2076affbc4", "size": 1124618, "path": "media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://1tfycz7y6gh34xr8dmgz32n94ny2xhcnjp4zqc7o1er0hor1af0likxq7jci958icda01m7kdo3qorwowfntkmmgfnc0wvu8k1hi29k4v3mimrtfbm26z3mp4facirq0/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://c9lnm5b0hf51vyjeqx3mvumndb832g3tr5r0xb20w90fmw05v477qx2dg86p778a522jo5aopb58xrvm8f4l7fty7z49hey3s1syxdhhs7ghpaaxek8u8q6ukp3y5hyk:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 35, "total_num_episodes": 35, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9969230591256979, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9084702878373503, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9208303614609197, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "f589dc3672fd4c0d3e6efc28c61bcb2e4d64bb9b16b8cfd10862ca30215bb34a", "size": 420, "artifact_path": "wandb-client-artifact://owo0dyxrre1slkbt2p5sdhn50wpbpfs0f9erbg41p5epi458xrr9ipefgpvax9goxnpui2az8ijqfy0r9ouj13cfa97sqjamdnpol8t8dky3oid0h3uuv6a4zrywmh2f/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://audqq590vi3tlowr5dz7ozst8685vl8361oj6nbrgvks13cl4zvkkotg2h0swurytvwgczvfp37q16mb8we9f3xy06dyfzlh3q4gm87kt44gzdiaye58uinhq7yef4wh:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 257}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.99692302942276, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.9084702730178833, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9208303689956665, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug-internal.log b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c450acc2d9f959787ee2eb5b248fdf085c49075f --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug-internal.log @@ -0,0 +1,760 @@ +2025-01-07 17:09:19,145 INFO StreamThr :2195200 [internal.py:wandb_internal():85] W&B internal server running at pid: 2195200, started at: 2025-01-07 17:09:19.144858 +2025-01-07 17:09:19,147 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:09:19,158 INFO WriterThread:2195200 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/run-nyyc47dd.wandb +2025-01-07 17:09:19,159 DEBUG SenderThread:2195200 [sender.py:send():379] send: header +2025-01-07 17:09:19,166 DEBUG SenderThread:2195200 [sender.py:send():379] send: run +2025-01-07 17:09:19,401 INFO SenderThread:2195200 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files +2025-01-07 17:09:19,401 INFO SenderThread:2195200 [sender.py:_start_run_threads():1188] run started: nyyc47dd with start time 1736287759.145573 +2025-01-07 17:09:19,417 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:09:19,417 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:09:19,463 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:09:19,468 DEBUG HandlerThread:2195200 [system_info.py:__init__():26] System info init +2025-01-07 17:09:19,468 DEBUG HandlerThread:2195200 [system_info.py:__init__():41] System info init done +2025-01-07 17:09:19,468 INFO HandlerThread:2195200 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:09:19,468 INFO SystemMonitor:2195200 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:09:19,468 INFO HandlerThread:2195200 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:09:19,469 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:09:19,469 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:09:19,470 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:09:19,470 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:09:19,471 INFO SystemMonitor:2195200 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:09:19,483 DEBUG HandlerThread:2195200 [system_info.py:probe():152] Probing system +2025-01-07 17:09:19,485 DEBUG HandlerThread:2195200 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:09:19,489 DEBUG HandlerThread:2195200 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:09:19,489 DEBUG HandlerThread:2195200 [system_info.py:probe():200] Probing system done +2025-01-07 17:09:19,489 DEBUG HandlerThread:2195200 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:09:19.483632', 'startedAt': '2025-01-07T22:09:19.131521', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.2671250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.973, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 736.9906234741211}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:09:19,489 INFO HandlerThread:2195200 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:09:19,489 INFO HandlerThread:2195200 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:09:19,490 DEBUG HandlerThread:2195200 [system_info.py:_save_code():44] Saving code +2025-01-07 17:09:19,495 DEBUG HandlerThread:2195200 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:09:19,495 DEBUG HandlerThread:2195200 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:09:19,558 DEBUG HandlerThread:2195200 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:09:19,558 INFO HandlerThread:2195200 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:09:19,561 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:09:19,561 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:09:19,562 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:09:19,562 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:09:19,802 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/ejfpsqxu-wandb-metadata.json +2025-01-07 17:09:19,804 INFO wandb-upload_1:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/86xoq2il-code/FISH/eval_robot.py +2025-01-07 17:09:19,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:09:19,815 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:09:19,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:09:19,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:09:19,818 INFO HandlerThread:2195200 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/tb" + save: true +} +control { + relay_id: "3d3bf609a5d34bbd83bc366daf72af5e" +} +_info { + stream_id: "nyyc47dd" +} + +2025-01-07 17:09:19,818 DEBUG HandlerThread:2195200 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:09:19,836 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/769jxegn-diff.patch +2025-01-07 17:09:19,867 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:19,970 DEBUG SenderThread:2195200 [sender.py:send():379] send: telemetry +2025-01-07 17:09:19,970 DEBUG SenderThread:2195200 [sender.py:send():379] send: tbrecord +2025-01-07 17:09:19,970 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:09:19,970 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 with policy live +2025-01-07 17:09:20,149 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/84npqobk-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:09:20,402 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH/eval_robot.py +2025-01-07 17:09:20,402 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/diff.patch +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-metadata.json +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code +2025-01-07 17:09:20,403 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH +2025-01-07 17:09:20,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:21,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:22,402 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:22,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:23,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:24,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:24,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:25,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:26,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:27,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:28,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:29,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:29,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:30,537 DEBUG SenderThread:2195200 [sender.py:send():379] send: config +2025-01-07 17:09:30,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:31,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:32,404 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:32,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:33,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:34,405 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:34,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:09:34,815 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:09:34,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:34,971 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:35,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:36,406 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:36,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:37,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:38,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:39,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:40,407 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:09:40,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:40,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:41,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:42,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:43,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:44,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:45,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:46,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:46,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:47,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:48,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:49,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:09:49,815 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:09:49,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:50,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:51,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:51,959 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:52,410 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml +2025-01-07 17:09:52,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:53,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:54,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:55,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:56,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:57,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:09:57,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:58,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:09:59,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:00,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:01,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:02,413 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:02,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:02,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:03,821 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:04,413 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:04,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:04,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:04,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:05,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:06,414 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:06,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:07,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:07,893 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:08,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:09,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:10,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:11,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:12,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:13,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:13,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:14,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:15,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:16,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:17,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:18,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:19,192 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:19,471 DEBUG SystemMonitor:2195200 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:10:19,472 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:10:19,815 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:19,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:19,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:20,418 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:20,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:21,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:22,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:23,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:24,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:24,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:25,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:26,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:27,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:28,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:29,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:29,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:30,828 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:31,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:32,567 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:10:32,567 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:10:32,567 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 with policy now +2025-01-07 17:10:32,569 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:10:32,569 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:10:32,569 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:10:32,788 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/2jrwafv1-media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:10:32,828 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:33,421 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:10:33,421 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:10:33,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media +2025-01-07 17:10:33,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:10:33,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos +2025-01-07 17:10:33,681 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/ydgcet1v-media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:10:33,828 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:34,422 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:34,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:34,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:34,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:34,982 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:35,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:36,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:37,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:38,423 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:38,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:39,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:40,424 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:40,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:40,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:41,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:42,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:43,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:44,425 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:10:44,681 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/e928zjcx-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:10:44,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:45,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:45,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:46,426 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:46,830 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:47,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:48,427 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:48,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:49,472 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:10:49,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:10:49,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:10:49,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:50,427 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:10:50,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:50,884 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:51,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:52,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:53,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:54,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:55,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:56,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:10:56,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:57,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:58,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:10:59,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:00,833 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:01,819 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:01,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:02,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:03,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:04,432 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:04,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:04,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:04,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:05,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:06,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:06,956 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:07,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:08,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:09,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:10,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:11,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:12,820 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:12,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:13,835 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:14,836 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:15,572 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:11:15,572 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 with policy now +2025-01-07 17:11:15,575 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:11:15,576 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:11:15,576 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:11:15,577 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:11:15,829 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/asbwh4g7-media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:11:15,836 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:11:16,436 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:11:16,711 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/wkaw6lqh-media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:11:16,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:17,844 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:18,596 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:18,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:19,473 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:11:19,823 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:19,823 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:19,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:20,437 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:20,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:21,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:22,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:23,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:23,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:24,439 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:24,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:25,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:26,250 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:11:26,251 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:11:26,251 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:11:26,439 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:11:26,440 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:11:26,440 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:26,839 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:27,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:28,440 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:28,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:29,247 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:29,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:30,441 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:30,840 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:31,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:32,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:33,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:34,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:34,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:34,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:34,932 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:35,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:36,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:37,841 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:38,444 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:38,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:39,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:40,822 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:40,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:41,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:42,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:43,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:44,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:45,826 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:45,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:46,446 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:46,842 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:47,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:48,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:49,473 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:11:49,816 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:11:49,816 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:11:49,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:50,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:50,896 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:51,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:52,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:53,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:54,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:55,843 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:56,202 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:11:56,844 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:57,643 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:11:57,644 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 with policy now +2025-01-07 17:11:57,649 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:11:57,650 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:11:57,650 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:11:57,650 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:11:57,849 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:57,867 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/n5jqxyys-media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:11:58,450 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:11:58,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:58,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:11:58,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:11:58,712 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/86nqclja-media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:11:58,849 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:11:59,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:11:59,846 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:00,451 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:00,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:01,229 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:01,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:02,452 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:02,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:03,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:04,825 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:04,828 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:04,849 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:05,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:06,453 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:06,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:06,964 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:07,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:08,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:09,850 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:10,072 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:12:10,072 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:12:10,073 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:12:10,455 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:12:10,455 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:12:10,727 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/g7tkup4a-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:12:10,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:11,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:12,455 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:12,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:12,853 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:13,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:14,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:15,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:16,457 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:16,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:17,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:18,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:18,851 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:19,473 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:12:19,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:19,818 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:19,865 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:20,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:21,852 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:22,855 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:23,855 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:23,906 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:24,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:25,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:26,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:27,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:28,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:29,679 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:29,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:30,462 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:30,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:31,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:32,856 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:33,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:34,817 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:34,818 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:34,861 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:34,961 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:35,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:36,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:37,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:38,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:39,857 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:40,676 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:40,860 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:41,743 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:12:41,743 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 with policy now +2025-01-07 17:12:41,750 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:12:41,750 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:12:41,750 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:12:41,751 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:12:41,860 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:41,976 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/uvvc2mgp-media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:12:42,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:12:42,741 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/31gn8yf8-media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:12:42,865 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:43,866 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:44,466 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:44,867 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:45,864 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:46,345 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:46,467 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:46,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:47,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:48,870 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:49,474 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:12:49,822 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:12:49,823 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:12:49,871 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:50,469 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:50,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:51,122 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:12:51,122 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:12:51,122 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:12:51,469 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:12:51,470 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:12:51,782 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:51,872 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:52,470 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:52,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:53,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:54,470 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:54,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:55,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:56,471 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:12:56,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:12:56,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:57,873 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:58,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:12:59,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:00,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:01,831 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:01,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:02,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:03,874 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:04,474 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:04,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:13:04,818 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:13:04,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:05,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:06,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:06,974 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:07,875 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:08,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:09,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:10,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:11,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:12,477 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:12,561 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:12,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:13,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:14,876 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:15,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:16,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:17,829 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:17,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:18,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:19,474 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:13:19,818 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:13:19,819 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:13:19,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:20,877 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:21,879 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:22,480 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:22,832 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:22,887 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:22,953 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:13:22,954 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:13:22,954 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 with policy now +2025-01-07 17:13:22,954 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:22,954 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:22,954 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:23,176 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/hlgu2p5z-media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:23,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval +2025-01-07 17:13:23,744 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/vwtjdzut-media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:23,881 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:24,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:24,893 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:25,481 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:25,887 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:26,886 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:27,893 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:28,482 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:28,505 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:28,888 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:29,892 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:30,483 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:30,903 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:31,902 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:32,901 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:33,834 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:33,896 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:34,827 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:13:34,827 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:13:34,898 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:35,485 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:35,614 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:13:35,614 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:13:35,717 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/734z8zcb-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:35,809 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:13:35,896 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:36,153 INFO wandb-upload_0:2195200 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpntcl4qjp +2025-01-07 17:13:36,181 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:13:36,187 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:13:36,485 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:36,485 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval +2025-01-07 17:13:36,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table +2025-01-07 17:13:36,537 INFO SenderThread:2195200 [sender.py:send_request_log_artifact():1525] logged artifact run-nyyc47dd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NTcwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5MTkyNg==', 'latestArtifact': None}} +2025-01-07 17:13:36,537 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:13:36,537 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 17:13:36,537 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:36,537 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,538 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,538 DEBUG SenderThread:2195200 [sender.py:send():379] send: telemetry +2025-01-07 17:13:36,538 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,538 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,538 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,539 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,539 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,539 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,539 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,540 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,540 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,540 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,540 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,541 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,541 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,541 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,541 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,542 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,542 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,542 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,542 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,543 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,543 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,543 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,543 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:36,543 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:36,543 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:36,544 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:13:36,708 INFO wandb-upload_1:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/bkk08nie-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:36,750 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/zftzjo7a-media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:36,934 INFO wandb-upload_1:2195200 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp27n0xk_n +2025-01-07 17:13:36,965 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/1eoim2ae-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:37,347 INFO SenderThread:2195200 [sender.py:send_request_log_artifact():1525] logged artifact run-nyyc47dd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NTc1Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5MTkyOQ==', 'latestArtifact': None}} +2025-01-07 17:13:37,347 DEBUG SenderThread:2195200 [sender.py:send():379] send: files +2025-01-07 17:13:37,348 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json with policy now +2025-01-07 17:13:37,348 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:37,348 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:37,348 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:37,349 DEBUG SenderThread:2195200 [sender.py:send():379] send: exit +2025-01-07 17:13:37,349 INFO SenderThread:2195200 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:13:37,349 INFO SenderThread:2195200 [sender.py:send_exit():588] handling runtime: 257 +2025-01-07 17:13:37,352 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:send_exit():594] send defer +2025-01-07 17:13:37,353 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,353 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:13:37,353 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:13:37,353 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,353 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:13:37,353 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:13:37,353 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:13:37,354 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:13:37,354 DEBUG SystemMonitor:2195200 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:13:37,354 DEBUG SystemMonitor:2195200 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:13:37,354 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:13:37,408 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:13:37,408 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:13:37,409 INFO HandlerThread:2195200 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:13:37,409 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,409 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:13:37,409 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:13:37,409 DEBUG SenderThread:2195200 [sender.py:send():379] send: stats +2025-01-07 17:13:37,409 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,409 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:13:37,410 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:37,410 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:13:37,410 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:13:37,410 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:37,410 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:13:37,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:37,592 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/b6dnw9qk-media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:38,486 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:40,487 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml +2025-01-07 17:13:44,181 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:44,182 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:44,182 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,183 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:13:44,183 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send():379] send: history +2025-01-07 17:13:44,183 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:13:44,183 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:44,184 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:13:44,185 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:44,185 DEBUG SenderThread:2195200 [sender.py:send():379] send: summary +2025-01-07 17:13:44,189 INFO SenderThread:2195200 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:13:44,190 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:13:44,190 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:44,190 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:13:44,190 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:13:44,190 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:13:44,190 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:13:44,190 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:44,190 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:13:44,191 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:44,191 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:13:44,489 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:44,514 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:45,455 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:13:45,456 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:45,456 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:45,456 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:13:45,456 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:45,456 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:13:45,456 INFO SenderThread:2195200 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:13:45,458 INFO SenderThread:2195200 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:13:45,460 INFO SenderThread:2195200 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:13:45,471 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:13:45,471 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:45,471 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:13:45,471 DEBUG SenderThread:2195200 [sender.py:send():379] send: artifact +2025-01-07 17:13:45,489 INFO Thread-12 :2195200 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:45,515 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:45,934 INFO wandb-upload_1:2195200 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmprtj2_75z +2025-01-07 17:13:45,936 INFO wandb-upload_2:2195200 [upload_job.py:push():85] Skipped uploading /tmp/tmpktrvvqs7/wandb-job.json +2025-01-07 17:13:46,045 INFO wandb-upload_0:2195200 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp3877ygx +2025-01-07 17:13:46,645 INFO SenderThread:2195200 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'versionIndex': 434}}} +2025-01-07 17:13:46,645 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:46,645 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:13:46,645 INFO SenderThread:2195200 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:13:47,490 INFO SenderThread:2195200 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files +2025-01-07 17:13:47,490 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml config.yaml +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json wandb-summary.json +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt requirements.txt +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:13:47,491 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:47,495 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log output.log +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/diff.patch diff.patch +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 media/videos/eval/3_eval_3_347a0bc393f70cbcbd15.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 media/videos/eval/1_eval_1_d5e6d8239c866ea812fc.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 media/videos/eval/4_eval_4_1a6b1bcbc82812416e36.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 media/videos/eval/0_eval_0_a1e0ba5727bcc50d3091.mp4 +2025-01-07 17:13:47,496 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 media/videos/eval/2_eval_2_83fca2ec84021031b7bd.mp4 +2025-01-07 17:13:47,499 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json media/table/eval/total_success_rate_ci_6_f589dc3672fd4c0d3e6e.table.json +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:13:47,500 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:47,500 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:47,500 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:13:47,500 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:13:47,500 INFO SenderThread:2195200 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:13:47,516 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:47,516 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:47,667 INFO wandb-upload_2:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/wandb-summary.json +2025-01-07 17:13:47,671 INFO wandb-upload_0:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/requirements.txt +2025-01-07 17:13:47,693 INFO wandb-upload_4:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/output.log +2025-01-07 17:13:47,705 INFO wandb-upload_1:2195200 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/files/config.yaml +2025-01-07 17:13:47,765 INFO wandb-upload_3:2195200 [upload_job.py:push():130] Uploaded file /tmp/tmpi7e06r7ywandb/gnwptkbk-events.out.tfevents.1736287759.leonmkim-ROG-Strix-G15CS-G15CS.2195027.0 +2025-01-07 17:13:47,966 INFO Thread-11 (_thread_body):2195200 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:13:47,966 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:47,966 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:13:47,966 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:47,966 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:13:47,966 INFO SenderThread:2195200 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:13:47,968 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:13:47,968 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:47,968 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:13:47,968 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:47,968 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:13:47,968 INFO SenderThread:2195200 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:13:48,142 INFO SenderThread:2195200 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:13:48,142 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:13:48,142 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:48,142 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:13:48,142 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:48,142 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:13:48,143 INFO SenderThread:2195200 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:13:48,143 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:13:48,143 INFO HandlerThread:2195200 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:13:48,143 DEBUG SenderThread:2195200 [sender.py:send():379] send: final +2025-01-07 17:13:48,143 DEBUG SenderThread:2195200 [sender.py:send():379] send: footer +2025-01-07 17:13:48,143 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: defer +2025-01-07 17:13:48,143 INFO SenderThread:2195200 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:13:48,144 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:48,144 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:13:48,144 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:48,144 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:13:48,145 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:13:48,145 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:13:48,147 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:13:48,147 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:13:48,194 DEBUG SenderThread:2195200 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:13:48,194 INFO MainThread:2195200 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:13:48,195 INFO MainThread:2195200 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:13:48,195 INFO MainThread:2195200 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:13:48,195 DEBUG HandlerThread:2195200 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:13:48,195 INFO HandlerThread:2195200 [handler.py:finish():882] shutting down handler +2025-01-07 17:13:49,146 INFO WriterThread:2195200 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/run-nyyc47dd.wandb +2025-01-07 17:13:49,194 INFO SenderThread:2195200 [sender.py:finish():1615] shutting down sender +2025-01-07 17:13:49,194 INFO SenderThread:2195200 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:13:49,194 INFO SenderThread:2195200 [file_pusher.py:join():175] waiting for file pusher diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug.log b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..eea2d805bdf799fff266fb7e08df42f74f4220f5 --- /dev/null +++ b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Configure stats pid to 2195027 +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/settings +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug.log +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/logs/debug-internal.log +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd'} +2025-01-07 17:09:19,138 INFO MainThread:2195027 [wandb_init.py:init():619] starting backend +2025-01-07 17:09:19,139 INFO MainThread:2195027 [wandb_init.py:init():623] setting up manager +2025-01-07 17:09:19,144 INFO MainThread:2195027 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:09:19,145 INFO MainThread:2195027 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:09:19,158 INFO MainThread:2195027 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:09:19,164 INFO MainThread:2195027 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:09:19,416 INFO MainThread:2195027 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:09:19,458 INFO MainThread:2195027 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:09:19,458 INFO MainThread:2195027 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:09:19,816 INFO MainThread:2195027 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:09:19,817 INFO MainThread:2195027 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:09:19,817 INFO MainThread:2195027 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/nyyc47dd/tb, True +2025-01-07 17:09:30,536 INFO MainThread:2195027 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:13:49,196 WARNING MsgRouterThr:2195027 [router.py:message_loop():77] message_loop has been closed diff --git a/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/run-nyyc47dd.wandb b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/run-nyyc47dd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d3691a9bf15ddcc4bfd2d75579669e869ab0629f Binary files /dev/null and b/nyyc47dd/wandb/run-20250107_170919-nyyc47dd/run-nyyc47dd.wandb differ diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7afd859d658fbb2e0c72000c049d039d19b267e5 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc9c87ebeed71a1995124399b1a701b4aa7f7b3e0cbc20d6e357470dd3ea210 +size 910102934 diff --git a/snapshot_13499.pt b/snapshot_13499.pt new file mode 100644 index 0000000000000000000000000000000000000000..e00dbdef53c3a91c81af8e5335bcc3b258701ee6 --- /dev/null +++ b/snapshot_13499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef39f83b9aa0d909336d5769bb92d860a4cdf5290395bfb3ec9e95aa9e0a0c3 +size 910102934 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..dec7315c4a864f74b78a7a13eb1409603ba43a4a --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387d241b2419435d8f1c381cc08d36f0514eb7dabf125d938dc74d9643ab3c6b +size 910102934 diff --git a/snapshot_13749.pt b/snapshot_13749.pt new file mode 100644 index 0000000000000000000000000000000000000000..13959de427e0ed990ca67cebc54e29e5ada38bae --- /dev/null +++ b/snapshot_13749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f8aef4b6cd8659ca440c8235424a5d5c40bc7c38560b32ec12efcf763315f8 +size 910102934 diff --git a/snapshot_13999.pt b/snapshot_13999.pt new file mode 100644 index 0000000000000000000000000000000000000000..480fe3ec69aa1544fa78e082572f1a6b373f74f5 --- /dev/null +++ b/snapshot_13999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d69f7c131d0ba7e0f902295ee19206819b8e311c583ffa3760fd17abe7f59b8 +size 910102934 diff --git a/snapshot_14249.pt b/snapshot_14249.pt new file mode 100644 index 0000000000000000000000000000000000000000..b12e72d775e89a46afc197c68c8ae22b98466b67 --- /dev/null +++ b/snapshot_14249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bc17375d90b2e0cf25deb743372ee69ce7960c57875220a96a9dd9dddbe983c +size 910102934 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..662533217186490ff55865b083f50dbd0a649a48 --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40dd8f87b993b2cac01fea97efd3f84b57326db023b500c1af07cd0b4634d3ff +size 910102934 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc72a667e5fd5cbc9994b1974246654883a0190e --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704e20faaaf8f820caae52052aefc9f52854e6b34a2e57a55701fac46aef87da +size 910102934 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ef700162edec708277d0a1f902f18230ff99b86 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed58c93907c87f4c3268e44a96fb69d4a4f67326270f7eeeb73c258069ec1b95 +size 910102934 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fa0970872b26954ef54fbe675201aade47f71f4 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:642a43416ca36fb8fa4b79be28d1db36d1c9a590196e4f7f092c40af52a6b538 +size 910102934 diff --git a/snapshot_7717.pt b/snapshot_7717.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f347e871a640f0c4131d2520d1cd8cf17dbb2ae --- /dev/null +++ b/snapshot_7717.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05dd23ee412c6446e89cdf7b82ec0b22c57a3e9c858d321e8ae5a010689b8acb +size 910102934 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..af1001c6dcef0df06d3c6435e9bf82a508be84a2 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff3f6b3ef2466228d338c0f62636d7211fba873cc4782259a9dcac836efc50e0 +size 910102934 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc72a667e5fd5cbc9994b1974246654883a0190e --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704e20faaaf8f820caae52052aefc9f52854e6b34a2e57a55701fac46aef87da +size 910102934 diff --git a/w3dz1sd1/episode_rosbags/aligned_depth_to_color_K.npy b/w3dz1sd1/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/w3dz1sd1/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/w3dz1sd1/episode_rosbags/cam_tf_world.npy b/w3dz1sd1/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/w3dz1sd1/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/w3dz1sd1/episode_rosbags/color_K.npy b/w3dz1sd1/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/w3dz1sd1/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/w3dz1sd1/episode_rosbags/depth_K.npy b/w3dz1sd1/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/w3dz1sd1/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/w3dz1sd1/eval_robot.log b/w3dz1sd1/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..7b5c021c9a451fceae8b128c72cbca358ba1832a --- /dev/null +++ b/w3dz1sd1/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:45:40,850][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:45:40,853][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/w3dz1sd1/wandb/debug-internal.log b/w3dz1sd1/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4569e918dd2df7c12e6556dfbc849d79819bf33e --- /dev/null +++ b/w3dz1sd1/wandb/debug-internal.log @@ -0,0 +1,780 @@ +2025-01-07 17:45:46,520 INFO StreamThr :2202979 [internal.py:wandb_internal():85] W&B internal server running at pid: 2202979, started at: 2025-01-07 17:45:46.520389 +2025-01-07 17:45:46,521 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:45:46,533 INFO WriterThread:2202979 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/run-w3dz1sd1.wandb +2025-01-07 17:45:46,533 DEBUG SenderThread:2202979 [sender.py:send():379] send: header +2025-01-07 17:45:46,541 DEBUG SenderThread:2202979 [sender.py:send():379] send: run +2025-01-07 17:45:46,844 INFO SenderThread:2202979 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files +2025-01-07 17:45:46,844 INFO SenderThread:2202979 [sender.py:_start_run_threads():1188] run started: w3dz1sd1 with start time 1736289946.520927 +2025-01-07 17:45:46,859 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:45:46,859 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:45:46,905 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:45:46,910 DEBUG HandlerThread:2202979 [system_info.py:__init__():26] System info init +2025-01-07 17:45:46,910 DEBUG HandlerThread:2202979 [system_info.py:__init__():41] System info init done +2025-01-07 17:45:46,910 INFO HandlerThread:2202979 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:45:46,910 INFO SystemMonitor:2202979 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:45:46,910 INFO HandlerThread:2202979 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:45:46,911 INFO SystemMonitor:2202979 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:45:46,912 INFO SystemMonitor:2202979 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:45:46,912 INFO SystemMonitor:2202979 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:45:46,912 INFO SystemMonitor:2202979 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:45:46,913 INFO SystemMonitor:2202979 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:45:46,923 DEBUG HandlerThread:2202979 [system_info.py:probe():152] Probing system +2025-01-07 17:45:46,925 DEBUG HandlerThread:2202979 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:45:46,929 DEBUG HandlerThread:2202979 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:45:46,929 DEBUG HandlerThread:2202979 [system_info.py:probe():200] Probing system done +2025-01-07 17:45:46,929 DEBUG HandlerThread:2202979 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:45:46.923320', 'startedAt': '2025-01-07T22:45:46.507480', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.48325, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.893, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 776.5426712036133}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:45:46,929 INFO HandlerThread:2202979 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:45:46,929 INFO HandlerThread:2202979 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:45:46,929 DEBUG HandlerThread:2202979 [system_info.py:_save_code():44] Saving code +2025-01-07 17:45:46,935 DEBUG HandlerThread:2202979 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:45:46,935 DEBUG HandlerThread:2202979 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:45:46,996 DEBUG HandlerThread:2202979 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:45:46,996 INFO HandlerThread:2202979 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:45:47,000 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:45:47,000 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:45:47,000 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:45:47,000 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:45:47,242 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:45:47,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:45:47,243 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:45:47,244 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/xxe974s3-wandb-metadata.json +2025-01-07 17:45:47,244 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:45:47,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:47,246 INFO HandlerThread:2202979 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/tb" + save: true +} +control { + relay_id: "0b5c39f7df294feaa5e841e15f34f16c" +} +_info { + stream_id: "w3dz1sd1" +} + +2025-01-07 17:45:47,246 DEBUG HandlerThread:2202979 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:45:47,280 INFO wandb-upload_1:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/04xdhoc8-code/FISH/eval_robot.py +2025-01-07 17:45:47,292 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/3vwzzu25-diff.patch +2025-01-07 17:45:47,408 DEBUG SenderThread:2202979 [sender.py:send():379] send: telemetry +2025-01-07 17:45:47,408 DEBUG SenderThread:2202979 [sender.py:send():379] send: tbrecord +2025-01-07 17:45:47,409 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:45:47,409 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 with policy live +2025-01-07 17:45:47,597 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/s6cmwij8-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:45:47,846 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:45:47,846 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH/eval_robot.py +2025-01-07 17:45:47,846 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt +2025-01-07 17:45:47,846 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/diff.patch +2025-01-07 17:45:47,846 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:45:47,846 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-metadata.json +2025-01-07 17:45:47,847 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH +2025-01-07 17:45:47,847 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code +2025-01-07 17:45:48,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:49,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:49,845 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:45:50,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:51,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:52,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:52,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:53,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:54,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:55,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:56,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:57,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:58,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:58,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:59,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:00,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:01,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:02,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:46:02,243 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:46:02,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:03,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:03,321 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:04,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:05,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:06,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:07,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:08,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:09,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:09,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:10,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:11,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:12,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:13,067 DEBUG SenderThread:2202979 [sender.py:send():379] send: config +2025-01-07 17:46:13,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:14,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:15,117 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:15,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:15,852 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:16,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:17,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:46:17,244 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:46:17,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:17,853 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:18,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:19,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:19,853 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:20,247 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:20,506 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:20,854 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml +2025-01-07 17:46:21,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:21,854 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:22,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:23,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:24,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:25,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:25,855 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:26,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:26,517 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:27,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:28,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:29,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:29,856 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:30,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:31,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:32,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:46:32,243 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:46:32,289 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:32,322 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:33,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:34,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:35,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:36,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:37,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:37,858 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:38,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:38,250 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:39,250 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:40,250 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:41,250 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:42,250 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:43,250 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:43,446 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:43,860 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:44,250 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:45,251 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:45,860 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:46,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:46,913 DEBUG SystemMonitor:2202979 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:46:46,914 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:46:47,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:46:47,243 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:46:47,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:48,252 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:49,253 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:49,351 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:50,252 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:51,253 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:52,253 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:53,253 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:54,253 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:54,944 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:46:55,155 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:46:55,155 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 with policy now +2025-01-07 17:46:55,155 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:46:55,156 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:46:55,156 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:46:55,158 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:46:55,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:55,410 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/zdx99p5c-media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 +2025-01-07 17:46:55,863 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:55,863 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 +2025-01-07 17:46:55,863 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:46:55,864 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval +2025-01-07 17:46:55,864 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos +2025-01-07 17:46:55,864 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media +2025-01-07 17:46:56,179 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/70zwrw9s-media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 +2025-01-07 17:46:56,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:57,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:57,864 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:46:58,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:59,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:46:59,865 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:00,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:00,909 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:01,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:02,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:47:02,244 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:47:02,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:03,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:03,866 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:04,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:05,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:06,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:06,397 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:07,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:08,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:09,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:10,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:11,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:12,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:12,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:13,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:14,255 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:15,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:16,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:16,914 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:47:17,243 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:47:17,244 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:47:17,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:17,318 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:17,870 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:47:18,130 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/3uotnjkj-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:47:18,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:19,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:19,871 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:20,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:21,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:22,257 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:23,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:23,257 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:23,872 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:24,257 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:25,257 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:26,257 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:27,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:28,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:28,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:29,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:30,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:31,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:31,875 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:32,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:47:32,244 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:47:32,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:33,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:33,333 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:33,876 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:34,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:35,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:36,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:37,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:38,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:39,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:39,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:40,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:41,260 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:42,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:43,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:44,246 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:44,260 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:45,260 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:46,260 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:46,915 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:47:47,244 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:47:47,244 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:47:47,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:48,260 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:49,268 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:49,350 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:49,393 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:47:49,394 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:47:49,394 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 with policy now +2025-01-07 17:47:49,394 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:47:49,394 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:47:49,394 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:47:49,681 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/t5qr5l35-media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 +2025-01-07 17:47:49,880 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:47:49,881 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:49,881 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 +2025-01-07 17:47:49,881 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval +2025-01-07 17:47:50,214 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/vfpade1g-media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 +2025-01-07 17:47:50,266 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:51,270 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:51,881 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:52,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:53,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:54,280 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:55,169 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:47:55,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:55,882 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:56,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:57,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:57,883 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:47:58,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:58,686 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:47:58,693 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:47:58,699 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:47:58,884 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:47:58,884 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:47:59,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:47:59,884 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:00,253 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:00,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:01,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:01,885 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:02,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:48:02,245 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:48:02,271 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:03,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:03,885 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:04,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:05,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:05,362 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:06,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:07,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:08,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:09,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:10,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:11,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:11,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:12,272 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:13,273 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:14,273 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:15,273 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:16,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:16,273 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:16,915 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:48:17,245 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:48:17,246 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:48:17,289 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:18,281 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:19,282 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:20,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:21,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:21,352 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:22,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:23,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:23,892 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:24,293 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:25,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:26,285 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:27,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:27,286 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:28,286 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:29,296 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:29,894 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:30,147 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:48:30,147 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 with policy now +2025-01-07 17:48:30,150 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:48:30,150 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:48:30,151 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:48:30,151 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:48:30,287 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:30,431 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/2xhxasnp-media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 +2025-01-07 17:48:30,894 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:48:30,894 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 +2025-01-07 17:48:30,894 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval +2025-01-07 17:48:31,224 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/2e8z4ic1-media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 +2025-01-07 17:48:31,290 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:31,894 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:32,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:32,258 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:48:32,258 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:48:32,301 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:33,294 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:34,297 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:35,297 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:35,896 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:36,295 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:37,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:37,312 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:37,896 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:38,299 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:39,306 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:39,515 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:48:39,515 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:48:39,515 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:48:39,897 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:48:39,897 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:48:40,153 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/azqmx253-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:48:40,300 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:41,311 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:41,898 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:42,264 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:42,303 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:43,330 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:44,317 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:45,317 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:45,899 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:46,317 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:46,916 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:48:47,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:48:47,248 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:48:47,317 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:47,344 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:48,317 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:49,317 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:50,318 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:51,319 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:51,901 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:52,319 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:53,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:53,319 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:54,319 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:55,319 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:55,903 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:56,319 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:57,319 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:57,903 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:48:58,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:48:59,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:48:59,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:00,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:01,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:02,248 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:49:02,248 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:49:02,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:03,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:04,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:04,385 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:05,320 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:06,321 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:07,321 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:08,321 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:09,321 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:10,242 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:10,321 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:11,333 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:11,908 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:12,131 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:49:12,131 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 with policy now +2025-01-07 17:49:12,135 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:49:12,136 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:49:12,136 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:49:12,136 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:49:12,325 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:12,406 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/yfj4yw0y-media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 +2025-01-07 17:49:12,908 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:49:12,909 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 +2025-01-07 17:49:12,909 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval +2025-01-07 17:49:13,247 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/x02skg7i-media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 +2025-01-07 17:49:13,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:13,909 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:14,332 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:14,909 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:15,254 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:15,332 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:16,336 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:16,916 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:49:17,252 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:49:17,252 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:49:17,338 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:17,910 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:18,338 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:19,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:19,911 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:20,336 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:20,725 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:21,337 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:21,912 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:49:22,341 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:22,568 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:49:22,568 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:49:22,569 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:49:22,912 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:49:23,339 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:23,912 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:24,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:25,344 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:25,741 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:26,339 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:27,339 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:27,914 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:28,339 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:29,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:30,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:31,259 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:31,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:32,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:49:32,249 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:49:32,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:33,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:34,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:35,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:36,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:36,407 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:37,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:38,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:39,340 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:40,341 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:41,341 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:42,256 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:42,341 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:43,341 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:44,341 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:45,341 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:46,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:46,917 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:49:47,249 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:49:47,250 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:49:47,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:47,353 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:48,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:49,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:50,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:51,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:52,342 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:53,044 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:53,357 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:53,924 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:54,345 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:54,513 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:49:54,513 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 with policy now +2025-01-07 17:49:54,520 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:49:54,521 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:49:54,521 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:49:54,521 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:49:54,789 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/8qa6tahn-media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 +2025-01-07 17:49:54,925 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:49:54,925 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 +2025-01-07 17:49:54,925 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval +2025-01-07 17:49:55,253 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/klnldrkk-media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 +2025-01-07 17:49:55,350 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:55,925 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:56,364 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:56,925 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:49:57,363 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:58,274 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:49:58,375 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:59,364 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:49:59,926 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:00,363 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:01,378 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:01,927 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:02,261 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:50:02,261 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:50:02,366 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:03,380 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:03,418 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:03,928 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:04,366 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:05,369 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:06,390 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:07,380 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:08,388 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:09,284 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:09,387 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:10,393 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:11,391 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:12,393 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:12,635 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:50:12,635 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:12,636 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:12,803 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:50:12,804 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:50:12,931 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:50:12,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:50:12,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:50:12,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media +2025-01-07 17:50:12,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table +2025-01-07 17:50:12,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval +2025-01-07 17:50:13,066 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:50:13,132 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/kooouc71-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:50:13,216 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/76qc2sny-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:50:13,369 INFO wandb-upload_1:2202979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0s0ujrp5 +2025-01-07 17:50:13,390 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:13,445 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:50:13,489 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:50:13,744 INFO SenderThread:2202979 [sender.py:send_request_log_artifact():1525] logged artifact run-w3dz1sd1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAwNDY5MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyMzIzOA==', 'latestArtifact': None}} +2025-01-07 17:50:13,744 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:50:13,744 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 17:50:13,744 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:50:13,744 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,745 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,745 DEBUG SenderThread:2202979 [sender.py:send():379] send: telemetry +2025-01-07 17:50:13,745 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,749 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,749 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,749 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,749 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,750 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,750 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,750 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,751 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,751 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,751 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,752 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,752 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,752 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,752 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,753 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,753 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,753 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,753 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,754 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,754 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:13,754 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:13,754 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:50:13,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:13,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:50:13,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:50:13,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json +2025-01-07 17:50:13,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval +2025-01-07 17:50:14,007 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/e994hzpj-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:50:14,167 INFO wandb-upload_1:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/fathqyru-media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json +2025-01-07 17:50:14,193 INFO wandb-upload_2:2202979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjmevjhc8 +2025-01-07 17:50:14,390 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:14,580 INFO SenderThread:2202979 [sender.py:send_request_log_artifact():1525] logged artifact run-w3dz1sd1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAwNDc5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyMzMxMw==', 'latestArtifact': None}} +2025-01-07 17:50:14,580 DEBUG SenderThread:2202979 [sender.py:send():379] send: files +2025-01-07 17:50:14,580 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json with policy now +2025-01-07 17:50:14,580 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:14,581 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:50:14,581 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:14,581 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:14,839 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/yg18e0j6-media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json +2025-01-07 17:50:14,932 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:50:15,391 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:15,933 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:16,391 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:16,917 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:50:17,251 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:50:17,251 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:50:17,391 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:17,809 DEBUG SenderThread:2202979 [sender.py:send():379] send: exit +2025-01-07 17:50:17,809 INFO SenderThread:2202979 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:50:17,810 INFO SenderThread:2202979 [sender.py:send_exit():588] handling runtime: 270 +2025-01-07 17:50:17,810 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:17,810 INFO SenderThread:2202979 [sender.py:send_exit():594] send defer +2025-01-07 17:50:17,810 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:17,810 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:50:17,810 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:17,810 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:50:17,810 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:50:17,810 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:17,811 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:50:17,811 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:17,811 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:50:17,811 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:50:17,811 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:17,811 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:50:17,811 INFO HandlerThread:2202979 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:50:17,811 INFO HandlerThread:2202979 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:50:17,811 DEBUG SystemMonitor:2202979 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:50:17,811 DEBUG SystemMonitor:2202979 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:50:17,812 INFO HandlerThread:2202979 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:50:17,889 INFO HandlerThread:2202979 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:50:17,889 INFO HandlerThread:2202979 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:50:17,889 INFO HandlerThread:2202979 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:50:17,890 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:17,890 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:50:17,890 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:50:17,890 DEBUG SenderThread:2202979 [sender.py:send():379] send: stats +2025-01-07 17:50:17,890 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:17,890 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:50:17,890 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:17,890 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:50:17,890 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:50:17,891 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:17,891 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:50:17,934 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:50:19,934 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:24,641 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:50:24,641 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:24,641 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:50:24,643 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:24,643 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:50:24,643 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:50:24,643 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:50:24,643 DEBUG SenderThread:2202979 [sender.py:send():379] send: history +2025-01-07 17:50:24,643 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:24,643 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:50:24,644 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:50:24,645 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:24,645 DEBUG SenderThread:2202979 [sender.py:send():379] send: summary +2025-01-07 17:50:24,649 INFO SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:50:24,649 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:24,649 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:50:24,649 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:50:24,649 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:24,649 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:50:24,650 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:24,650 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:50:24,653 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:50:24,774 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:50:24,774 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:24,774 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:50:24,775 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:24,775 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:50:24,811 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:50:24,936 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml +2025-01-07 17:50:24,937 INFO Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:50:26,055 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:50:26,055 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:50:26,055 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:26,055 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:50:26,055 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:26,055 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:50:26,055 INFO SenderThread:2202979 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:50:26,058 INFO SenderThread:2202979 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:50:26,060 INFO SenderThread:2202979 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:50:26,072 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:50:26,072 DEBUG SenderThread:2202979 [sender.py:send():379] send: artifact +2025-01-07 17:50:26,075 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:26,075 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:50:26,300 INFO SenderThread:2202979 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:50:26,301 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:26,301 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:50:26,301 INFO SenderThread:2202979 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:50:26,811 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:50:26,938 INFO SenderThread:2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:26,938 INFO SenderThread:2202979 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files +2025-01-07 17:50:26,938 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml config.yaml +2025-01-07 17:50:26,938 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json wandb-summary.json +2025-01-07 17:50:26,938 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:50:26,938 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt requirements.txt +2025-01-07 17:50:26,939 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:50:26,939 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log output.log +2025-01-07 17:50:26,939 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/diff.patch diff.patch +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json +2025-01-07 17:50:26,943 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:50:26,943 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:50:26,944 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:26,944 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:50:26,944 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:26,944 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:50:26,944 INFO SenderThread:2202979 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:50:27,133 INFO wandb-upload_1:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json +2025-01-07 17:50:27,166 INFO wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml +2025-01-07 17:50:27,174 INFO wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log +2025-01-07 17:50:27,196 INFO wandb-upload_3:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/szi5z8ek-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 +2025-01-07 17:50:27,199 INFO wandb-upload_4:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt +2025-01-07 17:50:27,399 INFO Thread-11 (_thread_body):2202979 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:50:27,399 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:27,399 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:50:27,399 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:27,399 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:50:27,400 INFO SenderThread:2202979 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:50:27,401 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:50:27,401 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:27,401 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:50:27,401 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:27,401 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:50:27,401 INFO SenderThread:2202979 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:50:27,613 INFO SenderThread:2202979 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:50:27,613 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:50:27,613 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:27,613 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:50:27,613 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:27,613 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:50:27,613 INFO SenderThread:2202979 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:50:27,613 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:50:27,613 INFO HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:50:27,614 DEBUG SenderThread:2202979 [sender.py:send():379] send: final +2025-01-07 17:50:27,614 DEBUG SenderThread:2202979 [sender.py:send():379] send: footer +2025-01-07 17:50:27,614 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: defer +2025-01-07 17:50:27,614 INFO SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:50:27,614 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:50:27,614 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:50:27,615 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:50:27,615 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:50:27,615 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:50:27,616 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:50:27,616 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:50:27,616 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:50:27,678 DEBUG SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:50:27,679 INFO MainThread:2202979 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:50:27,679 INFO MainThread:2202979 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:50:27,679 INFO MainThread:2202979 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:50:27,679 DEBUG HandlerThread:2202979 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:50:27,679 INFO HandlerThread:2202979 [handler.py:finish():882] shutting down handler +2025-01-07 17:50:28,615 INFO WriterThread:2202979 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/run-w3dz1sd1.wandb +2025-01-07 17:50:28,678 INFO SenderThread:2202979 [sender.py:finish():1615] shutting down sender +2025-01-07 17:50:28,679 INFO SenderThread:2202979 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:50:28,679 INFO SenderThread:2202979 [file_pusher.py:join():175] waiting for file pusher diff --git a/w3dz1sd1/wandb/debug.log b/w3dz1sd1/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0b7f672983b5dbe9dfa7f4d58d55799a31655ffa --- /dev/null +++ b/w3dz1sd1/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Configure stats pid to 2202791 +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/settings +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug.log +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug-internal.log +2025-01-07 17:45:46,514 INFO MainThread:2202791 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:45:46,515 INFO MainThread:2202791 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1'} +2025-01-07 17:45:46,515 INFO MainThread:2202791 [wandb_init.py:init():619] starting backend +2025-01-07 17:45:46,515 INFO MainThread:2202791 [wandb_init.py:init():623] setting up manager +2025-01-07 17:45:46,519 INFO MainThread:2202791 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:45:46,520 INFO MainThread:2202791 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:45:46,532 INFO MainThread:2202791 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:45:46,539 INFO MainThread:2202791 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:45:46,859 INFO MainThread:2202791 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:45:46,900 INFO MainThread:2202791 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:45:46,900 INFO MainThread:2202791 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:45:47,243 INFO MainThread:2202791 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:45:47,243 INFO MainThread:2202791 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:45:47,243 INFO MainThread:2202791 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:45:47,243 INFO MainThread:2202791 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:45:47,245 INFO MainThread:2202791 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:45:47,245 INFO MainThread:2202791 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/tb, True +2025-01-07 17:46:13,067 INFO MainThread:2202791 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 17:50:28,680 WARNING MsgRouterThr:2202791 [router.py:message_loop():77] message_loop has been closed diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dc51f568b395d0e7c1e5d4bc6c70a93c2e4a8b2 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736289946 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/diff.patch b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c0161c26d9b20c7c85242592d6833c2df9767742 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/diff.patch @@ -0,0 +1,20 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..aa5b9b9 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,13 +134,13 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' +-# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json new file mode 100644 index 0000000000000000000000000000000000000000..3e60606e4e031e793e54a4d981881fe8b4c01fc5 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[45, 45, 1.0, 0.9976060091453975, 1, 0.9280577673651977, 1, 0.9378636664766535, 1, 1, 1]]} \ No newline at end of file diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff873c8af303e057354f62cb3fc966b1a151a2a9 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e4ad6e0af44158052246a0a1373fdcc2d70c928fdb4e3f0cb15cde8937bf7a6 +size 1145682 diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5adcb78d487dc2e6d1fc4bfd8360f57a23e53a5b --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f499daa5972d95524f38e47699f47dbad979fd847f1d14020134898a7f3f7895 +size 1137042 diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a0ab87511bb9163f9a0ba04676996888c1d3c95 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77080ebc0cccad9e8ca80aae9493c38fe600db1f954e05dd099fb9bb3c7aa7a3 +size 1147997 diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0f5839e2a090362e7c28438aa0aaeae3a2e2ba09 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea44f1493fcbb8437821d3a0c5910e6052dd27bda35f92dfdb93658bec5efbbf +size 1156666 diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b90365762ac157d237556c8214b567fb54d4d327 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cadcc71b7662da48835a75b4c73eee2daf7d0a8e844c5da1154538d12f93b337 +size 1160393 diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ea5b9255118e10423ba0f54029d9650aac37c94a --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log @@ -0,0 +1,282 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736289973.067804]: resetting environment +[INFO] [1736289973.070916]: cleared current plan +[INFO] [1736289973.072074]: moving to home +[INFO] [1736289974.074448]: reached home +[INFO] [1736289974.075671]: reset action history +[INFO] [1736289976.782805]: environment reset +Starting episode 0 +[INFO] [1736289976.785476]: resetting environment +[INFO] [1736289976.789313]: cleared current plan +[INFO] [1736289976.790341]: moving to home +[INFO] [1736289977.792616]: reached home +[INFO] [1736289977.794207]: reset action history +[INFO] [1736289980.502462]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736289983.467120]: resetting environment +[INFO] [1736289983.469936]: cleared current plan +[INFO] [1736289983.470805]: moving to home +[INFO] [1736289984.473425]: reached home +[INFO] [1736289984.476359]: reset action history +[INFO] [1736289987.186944]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/episode_rosbags/episode_0_2025-01-07-17-46-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2202791.log +started roslaunch server http://158.130.50.37:41353/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2203083] +process[print_text-2]: started with pid [2203107] +process[rosbag_record-3]: started with pid [2203108] +started bagging! +For topic gripper_width: timestamp difference is 100662694 for nearest: 1736289995245491252 - target: 1736289995144828558 at idx 108 +For topic gripper_width: timestamp difference is 100997559 for nearest: 1736289995845491520 - target: 1736289995744493961 at idx 116 +For topic gripper_width: timestamp difference is 109582817 for nearest: 1736290002445490753 - target: 1736290002335907936 at idx 202 +For topic gripper_width: timestamp difference is 111621091 for nearest: 1736290003445443341 - target: 1736290003333822250 at idx 215 +[INFO] [1736290014.182684]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736290014.930646]: Stored episode 1. +[INFO] [1736290014.944559]: Saving video... +[INFO] [1736290015.136949]: Video saved! +Starting episode 1 +[INFO] [1736290015.156735]: resetting environment +[INFO] [1736290015.161414]: cleared current plan +[INFO] [1736290015.161600]: moving to home +[INFO] [1736290018.865262]: reached home +[INFO] [1736290018.865744]: reset action history +[INFO] [1736290021.572121]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290037.668934]: resetting environment +[INFO] [1736290037.673071]: cleared current plan +[INFO] [1736290037.673228]: moving to home +[INFO] [1736290038.674678]: reached home +[INFO] [1736290038.675088]: reset action history +[INFO] [1736290041.382053]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/episode_rosbags/episode_1_2025-01-07-17-47-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2202791.log +started roslaunch server http://158.130.50.37:33455/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2203218] +process[print_text-5]: started with pid [2203219] +process[rosbag_record-6]: started with pid [2203243] +started bagging! +For topic gripper_width: timestamp difference is 110830815 for nearest: 1736290050978803189 - target: 1736290050867972374 at idx 130 +[rosbag_record-6] killing on exitng episode...is 110830815 for nearest: 1736290050978803189 - target: 1736290050867972374 at idx 130 +[rosbag_record-6] killing on exitng episode...is 110830815 for nearest: 1736290050978803189 - target: 1736290050867972374 at idx 130 +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736290069.103659]: Stored episode 2. +[INFO] [1736290069.104510]: Saving video... +[INFO] [1736290069.348571]: Video saved! +Starting episode 2 +[INFO] [1736290069.382800]: resetting environment +[INFO] [1736290069.413421]: cleared current plan +[INFO] [1736290069.414399]: moving to home +[INFO] [1736290073.124418]: reached home +[INFO] [1736290073.133908]: reset action history +[INFO] [1736290075.862757]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290078.195941]: resetting environment +[INFO] [1736290078.204921]: cleared current plan +[INFO] [1736290078.206797]: moving to home +[INFO] [1736290079.208677]: reached home +[INFO] [1736290079.209105]: reset action history +[INFO] [1736290081.920351]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/episode_rosbags/episode_2_2025-01-07-17-48-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2202791.log +started roslaunch server http://158.130.50.37:42895/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2203364] +process[print_text-8]: started with pid [2203365] +process[rosbag_record-9]: started with pid [2203389] +started bagging! +For topic gripper_width: timestamp difference is 118151986 for nearest: 1736290101345490538 - target: 1736290101227338552 at idx 249 +[INFO] [1736290108.919643]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736290109.736677]: Stored episode 3. +[INFO] [1736290109.738489]: Saving video... +[INFO] [1736290110.110432]: Video saved! +Starting episode 3 +[INFO] [1736290110.150411]: resetting environment +[INFO] [1736290110.163291]: cleared current plan +[INFO] [1736290110.164805]: moving to home +[INFO] [1736290113.871410]: reached home +[INFO] [1736290113.876267]: reset action history +[INFO] [1736290116.654653]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290119.410427]: resetting environment +[INFO] [1736290119.427652]: cleared current plan +[INFO] [1736290119.430806]: moving to home +[INFO] [1736290120.436241]: reached home +[INFO] [1736290120.437264]: reset action history +[INFO] [1736290123.339463]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/episode_rosbags/episode_3_2025-01-07-17-48-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2202791.log +started roslaunch server http://158.130.50.37:40923/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2203534] +process[print_text-11]: started with pid [2203535] +process[rosbag_record-12]: started with pid [2203536] +started bagging! +For topic gripper_width: timestamp difference is 111982553 for nearest: 1736290130645490615 - target: 1736290130533508062 at idx 99 +For topic gripper_width: timestamp difference is 112410423 for nearest: 1736290130845489618 - target: 1736290130733079195 at idx 101 +For topic gripper_width: timestamp difference is 102330545 for nearest: 1736290133645455221 - target: 1736290133543124676 at idx 138 +For topic gripper_width: timestamp difference is 107163502 for nearest: 1736290136845513655 - target: 1736290136738350153 at idx 185 +For topic gripper_width: timestamp difference is -175677453 for nearest: 1736290149978823747 - target: 1736290150154501200 at idx 249 +[INFO] [1736290150.633497]: Storing episode... +[print_text-11] killing on exit +[pub_text-10] killing on exit[rosbag_record-12] killing on exit +[INFO] [1736290151.415785]: Stored episode 4. +[INFO] [1736290151.417670]: Saving video... +[INFO] [1736290152.080424]: Video saved! +Starting episode 4 +[INFO] [1736290152.145493]: resetting environment +[INFO] [1736290152.174540]: cleared current plan +[INFO] [1736290152.174782]: moving to home +[WARN] [1736290151.961851]: Plan exhausted +[WARN] [1736290152.004036]: Plan exhausted +[WARN] [1736290152.041852]: Plan exhausted +[WARN] [1736290152.084962]: Plan exhausted +[WARN] [1736290152.121810]: Plan exhausted +[WARN] [1736290152.162200]: Plan exhausted +[INFO] [1736290155.882688]: reached home +[INFO] [1736290155.891406]: reset action history +[INFO] [1736290158.681241]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290161.846818]: resetting environment +[INFO] [1736290161.874757]: cleared current plan +[INFO] [1736290161.876589]: moving to home +[INFO] [1736290162.881125]: reached home +[INFO] [1736290162.886397]: reset action history +[INFO] [1736290165.736840]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/episode_rosbags/episode_4_2025-01-07-17-49-25.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2202791.log +started roslaunch server http://158.130.50.37:42711/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2203670] +process[print_text-14]: started with pid [2203671] +process[rosbag_record-15]: started with pid [2203672] +started bagging! +[INFO] [1736290193.043411]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736290193.834419]: Stored episode 5. +[INFO] [1736290193.836035]: Saving video... +[INFO] [1736290194.456790]: Video saved! +[INFO] [1736290194.527837]: resetting environment +[INFO] [1736290194.560198]: cleared current plan +[INFO] [1736290194.570074]: moving to home +[WARN] [1736290194.363757]: Plan exhausted +[WARN] [1736290194.402438]: Plan exhausted +[WARN] [1736290194.442295]: Plan exhausted +[WARN] [1736290194.481810]: Plan exhausted +[WARN] [1736290194.531806]: Plan exhausted +[INFO] [1736290198.297819]: reached home +[INFO] [1736290198.298889]: reset action history +[INFO] [1736290201.168025]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-metadata.json b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ec1898a4a8e77fde7061114dbfd2d2ef927baa11 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:45:46.923320", + "startedAt": "2025-01-07T22:45:46.507480", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.48325, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.893, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 776.5426712036133 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..42990af9ffa29b49926d580d07a28f73bd00d171 --- /dev/null +++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "6e4ad6e0af44158052246a0a1373fdcc2d70c928fdb4e3f0cb15cde8937bf7a6", "size": 1145682, "path": "media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4"}, "global_step": 665, "_timestamp": 1736290213.3657737, "_runtime": 266.84484672546387, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "f499daa5972d95524f38e47699f47dbad979fd847f1d14020134898a7f3f7895", "size": 1137042, "path": "media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "77080ebc0cccad9e8ca80aae9493c38fe600db1f954e05dd099fb9bb3c7aa7a3", "size": 1147997, "path": "media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ea44f1493fcbb8437821d3a0c5910e6052dd27bda35f92dfdb93658bec5efbbf", "size": 1156666, "path": "media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "cadcc71b7662da48835a75b4c73eee2daf7d0a8e844c5da1154538d12f93b337", "size": 1160393, "path": "media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://xxfosas4lfd1qicq6hguqf2s932ygl1746pm0ucq1gi1f82fw132daaoalekjak6gjg3kcjr68ozgoiifx5i5flz2m4bqh90v6x11svpdz0kxdptl2ce3tgzruxk4hqg/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://yapwqn7lus3s9u08yb4y889dapuv8rrw9wd26f8pndu5yccon3p9gvrcm16uvve4pn7fuf8qa8bytuj4w4q25ayz13y4tpnf5olak7en3v4g5i9bzs7k9c219gmoi5ih:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 45, "total_num_episodes": 45, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9976060091453975, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9280577673651977, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9378636664766535, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "fedf8048f40139f691f96935e45ea221919d0af08d322a862760f1266f0fa949", "size": 420, "artifact_path": "wandb-client-artifact://iw1kxzjurxv9thlothfbycu0e1xxp43z6my3kkjpnehle2va79kvhwn5pu5ssykjxwei9n1xov5w3cn0w7jx97jnquj5802i1tfsxjt8akivcv4tsmsvkdod2e96vhzg/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ackhfx9f6rvjcyr0lsroe2cv3d7yenqijwgdec923602jrw16qgusp0dh8lzx1fjyre0rrj9ihin1x36shnq6hb3xngwwk5of0kj6p1egn9wyte6aowlguvf6ehgry5t:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 270}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9976059794425964, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.9280577898025513, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9378636479377747, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/run-w3dz1sd1.wandb b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/run-w3dz1sd1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..63af138b71d898c6a8bfbb384ac934f069bd3cc2 Binary files /dev/null and b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/run-w3dz1sd1.wandb differ