diff --git a/.gitattributes b/.gitattributes index 69f4004660a325f5ddb0bbbc08dbcc6cbf4af9a2..ba1efbc0bde31b289cea697de57ebb5f46bfe895 100644 --- a/.gitattributes +++ b/.gitattributes @@ -152,3 +152,33 @@ bbomq0jb/episode_rosbags/episode_1_2025-01-06-22-47-52.bag filter=lfs diff=lfs m 7332ua2l/episode_rosbags/episode_4_2025-01-06-18-04-19.bag filter=lfs diff=lfs merge=lfs -text 0a7h4b5r/episode_rosbags/episode_0_2025-01-06-20-11-17.bag filter=lfs diff=lfs merge=lfs -text 7332ua2l/episode_rosbags/episode_2_2025-01-06-18-02-38.bag filter=lfs diff=lfs merge=lfs -text +oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +oagunwyg/episode_rosbags/episode_3_2025-01-06-20-18-55.bag filter=lfs diff=lfs merge=lfs -text +73h7fwcu/episode_rosbags/episode_0_2025-01-07-16-40-16.bag filter=lfs diff=lfs merge=lfs -text +oagunwyg/episode_rosbags/episode_0_2025-01-06-20-16-24.bag filter=lfs diff=lfs merge=lfs -text +oagunwyg/episode_rosbags/episode_1_2025-01-06-20-17-12.bag filter=lfs diff=lfs merge=lfs -text +vhqg7j1c/episode_rosbags/episode_3_2025-01-06-17-58-15.bag filter=lfs diff=lfs merge=lfs -text +oagunwyg/episode_rosbags/episode_2_2025-01-06-20-17-59.bag filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9qvsp3e/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +zk6ipw9e/episode_rosbags/episode_3_2025-01-07-16-37-18.bag filter=lfs diff=lfs merge=lfs -text +zk6ipw9e/episode_rosbags/episode_2_2025-01-07-16-36-18.bag filter=lfs diff=lfs merge=lfs -text +zk6ipw9e/episode_rosbags/episode_4_2025-01-07-16-38-21.bag filter=lfs diff=lfs merge=lfs -text +zk6ipw9e/episode_rosbags/episode_1_2025-01-07-16-35-28.bag filter=lfs diff=lfs merge=lfs -text diff --git a/2dfvmjp7/.hydra/config.yaml b/2dfvmjp7/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/2dfvmjp7/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/2dfvmjp7/.hydra/hydra.yaml b/2dfvmjp7/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..601a42c45797ecdbbc0bc367ab16a8494843d56c --- /dev/null +++ b/2dfvmjp7/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/2dfvmjp7 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/2dfvmjp7/.hydra/overrides.yaml b/2dfvmjp7/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/2dfvmjp7/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/2dfvmjp7/eval_policy.log b/2dfvmjp7/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..e2e4b834e23bf0600a76160df961dd1a76a1e459 --- /dev/null +++ b/2dfvmjp7/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 17:55:16,652][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:55:16,655][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 17:55:21,140][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/73h7fwcu/episode_rosbags/episode_0_2025-01-07-16-40-16.bag b/73h7fwcu/episode_rosbags/episode_0_2025-01-07-16-40-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..22f72966f708ded1165024c22252125c88a3ebc4 --- /dev/null +++ b/73h7fwcu/episode_rosbags/episode_0_2025-01-07-16-40-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20acf85753255f085227e80fc81e869be46695e7dc4950cbf00ebc9bda153957 +size 2611899773 diff --git a/k9qvsp3e/.hydra/config.yaml b/k9qvsp3e/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/k9qvsp3e/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/k9qvsp3e/.hydra/hydra.yaml b/k9qvsp3e/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df21580a11279574a0812bff9598ebcbd8f76ab6 --- /dev/null +++ b/k9qvsp3e/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/k9qvsp3e/.hydra/overrides.yaml b/k9qvsp3e/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/k9qvsp3e/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/k9qvsp3e/episode_rosbags/aligned_depth_to_color_K.npy b/k9qvsp3e/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/k9qvsp3e/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/k9qvsp3e/episode_rosbags/cam_tf_world.npy b/k9qvsp3e/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/k9qvsp3e/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/k9qvsp3e/episode_rosbags/color_K.npy b/k9qvsp3e/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/k9qvsp3e/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/k9qvsp3e/episode_rosbags/depth_K.npy b/k9qvsp3e/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/k9qvsp3e/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/k9qvsp3e/eval_robot.log b/k9qvsp3e/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..578397e9ef77fd043c7f3607621deece4fafba44 --- /dev/null +++ b/k9qvsp3e/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 22:40:16,044][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:40:16,047][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/k9qvsp3e/eval_video/0_eval.mp4 b/k9qvsp3e/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..03ec7d633c6bcf9362f279a413113a61585f90ac --- /dev/null +++ b/k9qvsp3e/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9451f95a8b57a292306a7c34fc4be83de83308d1b1c9ced5c4f379a10cac6d6 +size 1556320 diff --git a/k9qvsp3e/eval_video/1_eval.mp4 b/k9qvsp3e/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0648ea2e83db0cf202223cd8d28d63d31dcf1730 --- /dev/null +++ b/k9qvsp3e/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f87c6c3fd8435e0a2b8e7945b3446fdc15074445638cc05f150d2457751fb783 +size 1546640 diff --git a/k9qvsp3e/eval_video/2_eval.mp4 b/k9qvsp3e/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..43dd71faf87babfc57f58549038a77e8a6527585 --- /dev/null +++ b/k9qvsp3e/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b784a43e185aa1b9230692fde621c244ef7c50bcd3e949d1ed1f03a5f2233ec0 +size 1567947 diff --git a/k9qvsp3e/eval_video/3_eval.mp4 b/k9qvsp3e/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f69330b84a5e774ed16d949950c977bc12f40640 --- /dev/null +++ b/k9qvsp3e/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ab2150c981107a3ff2e2f4528cdca9b37326f11a936bffb4fd024e86bebd02 +size 1597568 diff --git a/k9qvsp3e/eval_video/4_eval.mp4 b/k9qvsp3e/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a31b0d592c66a50a5a42665c5fd674870a80393b --- /dev/null +++ b/k9qvsp3e/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb5c8e77d3595ea5b2040755420ab4567057c833edbc27da6ec7c1b21056b47 +size 1562396 diff --git a/k9qvsp3e/tb/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 b/k9qvsp3e/tb/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 new file mode 100644 index 0000000000000000000000000000000000000000..b8b59dc37076130f058382f75019cc9fb4c19c29 --- /dev/null +++ b/k9qvsp3e/tb/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:624d9ffcd940534a320d4a6a2503caffa2193ccda7f44508d946c6874b9bf891 +size 6539 diff --git a/k9qvsp3e/wandb/debug-internal.log b/k9qvsp3e/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d7dfdf3a264c2c0aa0b393c4b8a5d4f666fe770d --- /dev/null +++ b/k9qvsp3e/wandb/debug-internal.log @@ -0,0 +1,883 @@ +2025-01-06 22:40:21,753 INFO StreamThr :2128784 [internal.py:wandb_internal():85] W&B internal server running at pid: 2128784, started at: 2025-01-06 22:40:21.752826 +2025-01-06 22:40:21,753 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:40:21,766 INFO WriterThread:2128784 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/run-k9qvsp3e.wandb +2025-01-06 22:40:21,766 DEBUG SenderThread:2128784 [sender.py:send():379] send: header +2025-01-06 22:40:21,774 DEBUG SenderThread:2128784 [sender.py:send():379] send: run +2025-01-06 22:40:22,079 INFO SenderThread:2128784 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files +2025-01-06 22:40:22,079 INFO SenderThread:2128784 [sender.py:_start_run_threads():1188] run started: k9qvsp3e with start time 1736221221.753591 +2025-01-06 22:40:22,098 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:40:22,098 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:40:22,172 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:40:22,176 DEBUG HandlerThread:2128784 [system_info.py:__init__():26] System info init +2025-01-06 22:40:22,176 DEBUG HandlerThread:2128784 [system_info.py:__init__():41] System info init done +2025-01-06 22:40:22,176 INFO HandlerThread:2128784 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:40:22,176 INFO SystemMonitor:2128784 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:40:22,177 INFO HandlerThread:2128784 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:40:22,177 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:40:22,177 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:40:22,178 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:40:22,179 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:40:22,179 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:40:22,190 DEBUG HandlerThread:2128784 [system_info.py:probe():152] Probing system +2025-01-06 22:40:22,191 DEBUG HandlerThread:2128784 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_info.py:probe():200] Probing system done +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:40:22.190152', 'startedAt': '2025-01-07T03:40:21.742569', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.344375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.012, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 602.1890144348145}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:40:22,196 INFO HandlerThread:2128784 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:40:22,196 INFO HandlerThread:2128784 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_info.py:_save_code():44] Saving code +2025-01-06 22:40:22,197 DEBUG HandlerThread:2128784 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:40:22,197 DEBUG HandlerThread:2128784 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:40:22,260 DEBUG HandlerThread:2128784 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:40:22,260 INFO HandlerThread:2128784 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:40:22,263 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:40:22,264 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:40:22,264 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:40:22,264 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:40:22,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:40:22,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:40:22,507 DEBUG SenderThread:2128784 [sender.py:send():379] send: telemetry +2025-01-06 22:40:22,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:22,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:40:22,509 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:40:22,522 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/bla36jn6-wandb-metadata.json +2025-01-06 22:40:22,526 INFO wandb-upload_2:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/qiopqikn-diff.patch +2025-01-06 22:40:22,553 INFO HandlerThread:2128784 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/tb" + save: true +} +control { + relay_id: "7bd0dea66d0146f086ecefe898319b5c" +} +_info { + stream_id: "k9qvsp3e" +} + +2025-01-06 22:40:22,553 DEBUG HandlerThread:2128784 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:40:22,582 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/01kaoati-code/FISH/eval_robot.py +2025-01-06 22:40:22,603 DEBUG SenderThread:2128784 [sender.py:send():379] send: tbrecord +2025-01-06 22:40:22,604 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:40:22,604 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 with policy live +2025-01-06 22:40:22,796 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/s4cv1hl3-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH/eval_robot.py +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/diff.patch +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-metadata.json +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH +2025-01-06 22:40:23,082 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code +2025-01-06 22:40:23,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:24,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:25,080 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:25,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:26,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:27,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:27,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:28,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:29,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:30,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:31,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:32,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:33,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:33,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:34,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:35,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:35,633 DEBUG SenderThread:2128784 [sender.py:send():379] send: config +2025-01-06 22:40:36,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:37,083 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:37,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:40:37,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:40:37,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:38,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:38,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:39,084 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:39,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:40,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:41,084 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:41,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:42,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:43,085 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:43,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:44,125 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:44,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:45,086 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:45,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:46,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:47,086 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:47,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:48,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:49,087 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:49,320 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:49,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:50,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:51,088 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:51,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:52,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:40:52,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:40:52,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:53,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:54,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:54,584 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:55,089 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml +2025-01-06 22:40:55,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:56,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:57,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:58,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:59,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:00,105 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:00,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:01,090 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:01,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:02,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:03,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:04,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:05,305 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:05,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:06,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:07,092 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:07,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:07,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:07,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:08,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:09,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:10,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:10,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:11,093 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:11,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:12,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:13,093 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:13,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:14,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:15,094 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:15,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:16,009 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:16,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:17,095 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:17,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:18,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:19,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:20,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:21,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:21,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:22,180 DEBUG SystemMonitor:2128784 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:41:22,180 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:41:22,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:22,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:22,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:23,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:24,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:25,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:25,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:41:25,573 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:41:25,574 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 with policy now +2025-01-06 22:41:25,577 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:41:25,577 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:41:25,577 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:41:25,839 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/bz6t9kbj-media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:41:26,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:41:26,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:41:26,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media +2025-01-06 22:41:26,098 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:41:26,098 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos +2025-01-06 22:41:26,374 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/j41ugirl-media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:41:26,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:26,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:27,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:27,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:28,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:29,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:30,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:31,099 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:31,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:31,785 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:32,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:33,100 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:33,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:34,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:35,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:36,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:37,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:37,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:37,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:37,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:38,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:39,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:40,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:41,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:42,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:43,072 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:43,103 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:41:43,340 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/kt4aldiq-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:41:43,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:44,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:45,103 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:45,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:46,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:47,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:48,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:48,664 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:49,105 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:49,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:50,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:51,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:52,185 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:41:52,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:52,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:52,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:53,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:54,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:54,613 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:55,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:56,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:57,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:58,523 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:59,522 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:00,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:00,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:01,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:02,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:03,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:04,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:05,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:05,526 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:06,526 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:07,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:07,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:07,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:08,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:09,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:10,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:10,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:11,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:12,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:13,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:14,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:15,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:16,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:16,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:17,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:18,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:19,114 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:19,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:20,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:21,115 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:21,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:21,729 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:22,181 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:42:22,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:22,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:22,531 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:23,384 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:42:23,385 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 with policy now +2025-01-06 22:42:23,387 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:42:23,388 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:42:23,388 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:42:23,388 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:42:23,530 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:23,646 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/tvpuqpr1-media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:42:24,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:42:24,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:42:24,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:42:24,453 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/jw837zy1-media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:42:24,533 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:25,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:25,533 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:26,545 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:26,917 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:27,544 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:28,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:29,118 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:29,547 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:30,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:31,118 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:31,539 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:32,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:32,540 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:33,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:34,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:35,545 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:36,548 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:37,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:37,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:37,518 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:37,561 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:38,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:39,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:40,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:41,554 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:42,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:42,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:43,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:44,552 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:45,556 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:46,551 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:47,552 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:48,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:48,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:49,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:50,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:51,556 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:52,182 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:42:52,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:52,515 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:52,558 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:53,559 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:53,596 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:54,560 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:55,561 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:56,562 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:57,562 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:58,563 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:59,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:59,570 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:00,563 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:01,570 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:02,567 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:03,567 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:04,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:04,569 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:05,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:06,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:07,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:07,511 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:07,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:08,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:09,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:09,638 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:10,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:11,576 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:12,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:13,574 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:14,576 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:15,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:15,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:16,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:17,577 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:18,038 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:43:18,038 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:43:18,039 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:43:18,133 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:43:18,134 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:43:18,369 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/7v08gmdz-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:43:18,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:19,133 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:19,580 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:20,588 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:21,134 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:21,141 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:21,584 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:22,182 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:43:22,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:22,510 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:22,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:23,135 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:23,581 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:24,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:25,136 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:25,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:26,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:27,073 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:27,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:28,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:29,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:30,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:31,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:32,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:32,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:33,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:34,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:35,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:36,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:37,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:37,514 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:37,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:37,597 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:38,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:39,588 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:40,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:41,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:42,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:43,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:43,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:44,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:45,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:46,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:47,591 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:48,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:48,588 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:49,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:50,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:51,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:52,183 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:43:52,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:52,518 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:52,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:53,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:53,591 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:54,590 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:55,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:56,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:57,590 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:58,460 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:43:58,460 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 with policy now +2025-01-06 22:43:58,465 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:43:58,466 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:43:58,466 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:43:58,466 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:43:58,590 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:58,720 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/g1wi6unv-media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:43:59,147 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:43:59,147 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:59,148 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:43:59,148 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:43:59,450 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/wezinnen-media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:43:59,483 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:59,595 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:00,596 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:01,148 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:01,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:02,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:03,149 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:03,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:04,536 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:04,605 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:05,601 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:06,600 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:07,150 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:07,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:07,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:07,611 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:08,602 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:09,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:09,607 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:10,608 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:11,609 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:12,612 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:13,605 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:14,610 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:15,538 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:15,612 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:16,612 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:17,614 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:18,615 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:19,616 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:20,154 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:44:20,398 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/awyasg86-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:44:20,619 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:20,890 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:44:20,891 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:44:20,891 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:20,891 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:44:21,154 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:44:21,155 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:21,619 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:22,183 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:44:22,522 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:22,522 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:22,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:23,155 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:23,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:24,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:25,156 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:25,619 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:25,902 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:26,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:27,157 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:27,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:28,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:29,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:30,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:31,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:31,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:32,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:33,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:34,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:35,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:36,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:36,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:37,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:37,517 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:37,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:38,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:39,161 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:39,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:40,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:41,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:41,937 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:42,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:43,162 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:43,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:44,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:45,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:46,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:47,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:47,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:48,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:49,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:50,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:51,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:52,184 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:44:52,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:52,518 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:52,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:52,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:53,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:54,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:55,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:56,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:57,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:58,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:58,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:59,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:00,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:01,522 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:45:01,522 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 with policy now +2025-01-06 22:45:01,525 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:45:01,526 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:45:01,526 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:45:01,526 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:45:01,626 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:01,787 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/48s1juc4-media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:45:02,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:45:02,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:45:02,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:45:02,467 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/tm6tn434-media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:45:02,635 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:03,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:03,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:03,630 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:04,631 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:05,641 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:06,643 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:07,170 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:07,522 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:07,522 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:07,643 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:08,647 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:08,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:09,171 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:09,654 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:10,655 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:11,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:12,656 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:12,953 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:45:12,954 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:45:12,955 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:45:13,172 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:45:13,173 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:45:13,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:13,999 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:14,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:15,173 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:15,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:16,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:17,665 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:18,656 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:19,174 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:19,353 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:19,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:20,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:21,175 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:21,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:22,184 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:45:22,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:22,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:22,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:23,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:24,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:24,665 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:25,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:26,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:27,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:28,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:29,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:30,541 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:30,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:31,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:32,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:33,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:34,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:35,541 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:35,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:36,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:37,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:37,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:37,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:38,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:39,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:40,635 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:40,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:41,182 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:41,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:42,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:43,183 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:43,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:44,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:45,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:46,541 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:46,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:47,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:48,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:49,663 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:50,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:51,545 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:51,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:52,184 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:45:52,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:52,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:52,663 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:53,667 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:54,418 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:45:54,418 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 with policy now +2025-01-06 22:45:54,423 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:45:54,423 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:45:54,423 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:45:54,434 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:45:54,665 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:54,721 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/9nb25diq-media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:45:55,471 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/19zishh9-media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:45:55,674 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:56,676 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:57,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:57,488 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:57,681 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:58,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:59,188 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:59,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:00,680 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:01,682 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:02,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:46:02,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:03,016 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:03,016 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,017 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,118 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:46:03,119 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval +2025-01-06 22:46:03,301 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:46:03,392 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/72s13plb-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:03,432 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/2k8grycg-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:03,607 INFO wandb-upload_2:2128784 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpm0rttqun +2025-01-06 22:46:03,677 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:03,701 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:46:03,745 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:46:03,988 INFO SenderThread:2128784 [sender.py:send_request_log_artifact():1525] logged artifact run-k9qvsp3e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTQxNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2MDI0NQ==', 'latestArtifact': None}} +2025-01-06 22:46:03,988 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:46:03,988 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json with policy now +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,989 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send():379] send: telemetry +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,990 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,990 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,990 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,990 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,992 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,993 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,993 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,993 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,993 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,994 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,994 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,994 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,994 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,995 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,995 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,995 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,995 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,995 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,996 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,996 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,996 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,996 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval +2025-01-06 22:46:04,234 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/x7qt8ify-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:04,369 INFO wandb-upload_2:2128784 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk36k0l6r +2025-01-06 22:46:04,382 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/ggch35pc-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:04,677 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:04,780 INFO SenderThread:2128784 [sender.py:send_request_log_artifact():1525] logged artifact run-k9qvsp3e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTQ0MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2MDI0Ng==', 'latestArtifact': None}} +2025-01-06 22:46:04,780 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:46:04,780 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json with policy now +2025-01-06 22:46:04,780 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:04,781 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:04,781 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:05,029 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/4up7fth4-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:05,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:05,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:05,677 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:05,805 DEBUG SenderThread:2128784 [sender.py:send():379] send: exit +2025-01-06 22:46:05,805 INFO SenderThread:2128784 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:send_exit():588] handling runtime: 343 +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:send_exit():594] send defer +2025-01-06 22:46:05,806 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,806 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:46:05,806 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:46:05,806 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,806 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:46:05,807 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,807 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:46:05,807 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:46:05,807 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,807 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:46:05,807 INFO HandlerThread:2128784 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:46:05,807 DEBUG SystemMonitor:2128784 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:46:05,807 DEBUG SystemMonitor:2128784 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:46:05,808 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:46:05,808 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:46:05,859 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:46:05,859 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:46:05,859 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:46:05,860 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:46:05,860 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:46:05,860 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,860 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:46:05,860 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:46:05,860 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,861 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:46:06,192 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:07,192 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:12,022 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:12,022 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:46:12,022 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:46:12,024 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,024 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:46:12,024 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:46:12,024 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:12,024 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:12,024 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:46:12,024 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:12,025 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:12,025 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:12,025 DEBUG SenderThread:2128784 [sender.py:send():379] send: summary +2025-01-06 22:46:12,029 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:12,029 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,029 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:46:12,029 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:46:12,029 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:12,029 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:46:12,030 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,030 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:46:12,033 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:46:12,160 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:46:12,160 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:12,160 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:46:12,161 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,161 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:46:12,194 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:12,195 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml +2025-01-06 22:46:12,807 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:13,102 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:46:13,102 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:13,102 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:13,102 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:46:13,102 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:13,102 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:46:13,102 INFO SenderThread:2128784 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:46:13,105 INFO SenderThread:2128784 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:46:13,107 INFO SenderThread:2128784 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:46:13,122 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:46:13,122 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:13,122 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:46:13,123 DEBUG SenderThread:2128784 [sender.py:send():379] send: artifact +2025-01-06 22:46:13,195 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:13,493 INFO wandb-upload_1:2128784 [upload_job.py:push():85] Skipped uploading /tmp/tmp_60dcf4r/wandb-job.json +2025-01-06 22:46:13,493 INFO wandb-upload_2:2128784 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuv317_7z +2025-01-06 22:46:13,612 INFO wandb-upload_0:2128784 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgpe_9mkz +2025-01-06 22:46:13,807 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:14,307 INFO SenderThread:2128784 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTc3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'versionIndex': 428}}} +2025-01-06 22:46:14,307 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:14,307 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:46:14,307 INFO SenderThread:2128784 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml config.yaml +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json wandb-summary.json +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt requirements.txt +2025-01-06 22:46:15,197 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:46:15,199 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:15,200 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log output.log +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/diff.patch diff.patch +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:46:15,203 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:15,203 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:15,203 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:46:15,203 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:15,206 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,206 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:46:15,206 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,206 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:46:15,206 INFO SenderThread:2128784 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:46:15,369 INFO wandb-upload_2:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:15,386 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml +2025-01-06 22:46:15,386 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt +2025-01-06 22:46:15,456 INFO wandb-upload_4:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:15,490 INFO wandb-upload_3:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/5ac8u05j-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:15,691 INFO Thread-11 (_thread_body):2128784 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:46:15,691 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,691 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:46:15,691 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,691 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:46:15,692 INFO SenderThread:2128784 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:46:15,694 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:46:15,694 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,695 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:46:15,695 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,695 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:46:15,695 INFO SenderThread:2128784 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:46:15,750 INFO SenderThread:2128784 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:46:15,751 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:46:15,751 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,751 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:46:15,751 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,751 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:46:15,751 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:46:15,751 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,751 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send():379] send: final +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send():379] send: footer +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,752 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:46:15,752 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:15,753 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:46:15,753 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:15,753 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:46:15,754 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:46:15,755 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:46:15,755 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:15,807 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:15,807 INFO MainThread:2128784 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:46:15,808 INFO MainThread:2128784 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:46:15,808 INFO MainThread:2128784 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:46:15,808 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:46:15,808 INFO HandlerThread:2128784 [handler.py:finish():882] shutting down handler +2025-01-06 22:46:16,753 INFO WriterThread:2128784 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/run-k9qvsp3e.wandb +2025-01-06 22:46:16,807 INFO SenderThread:2128784 [sender.py:finish():1615] shutting down sender +2025-01-06 22:46:16,807 INFO SenderThread:2128784 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:46:16,807 INFO SenderThread:2128784 [file_pusher.py:join():175] waiting for file pusher diff --git a/k9qvsp3e/wandb/debug.log b/k9qvsp3e/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..da1e0e18578083c6e9497bd1736c9a1c46fab0b0 --- /dev/null +++ b/k9qvsp3e/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Configure stats pid to 2128589 +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/settings +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug.log +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug-internal.log +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e'} +2025-01-06 22:40:21,747 INFO MainThread:2128589 [wandb_init.py:init():619] starting backend +2025-01-06 22:40:21,747 INFO MainThread:2128589 [wandb_init.py:init():623] setting up manager +2025-01-06 22:40:21,752 INFO MainThread:2128589 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:40:21,752 INFO MainThread:2128589 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:40:21,765 INFO MainThread:2128589 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:40:21,772 INFO MainThread:2128589 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:40:22,097 INFO MainThread:2128589 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:40:22,166 INFO MainThread:2128589 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:40:22,166 INFO MainThread:2128589 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:40:22,509 INFO MainThread:2128589 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:40:22,509 INFO MainThread:2128589 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/tb, True +2025-01-06 22:40:35,633 INFO MainThread:2128589 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:46:16,809 WARNING MsgRouterThr:2128589 [router.py:message_loop():77] message_loop has been closed diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH/eval_robot.py b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fb3a7368a5962677fc23b7f27c5372f28b1134c --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1013_1' +wandb_notes: + desc: null + value: 1013_1_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736221221 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/diff.patch b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..65db0437f2055dbb409cbea805d077d23c6305eb --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..1c0be7a 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d1a9a8f5261253760795e8c0f15e63e65e093e60 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.09420213111410328, 0.8653537630304143, 0.08973243840793546, 0.8341690897273251, 0.12645963586887082, 0.7050612568445163, 0.11327369891391026, 0.6491125076710976]]} \ No newline at end of file diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9d792d4006105450f6870c1e1f2a1e18fb615772 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[22, 25, 0.88, 0.7029817103933635, 0.9734671934492175, 0.6976236402593188, 0.9621387421160531, 0.7217742651257426, 0.9598798117611944, 0.7949399984783139, 0.9499553627365946]]} \ No newline at end of file diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..03ec7d633c6bcf9362f279a413113a61585f90ac --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9451f95a8b57a292306a7c34fc4be83de83308d1b1c9ced5c4f379a10cac6d6 +size 1556320 diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0648ea2e83db0cf202223cd8d28d63d31dcf1730 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f87c6c3fd8435e0a2b8e7945b3446fdc15074445638cc05f150d2457751fb783 +size 1546640 diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..43dd71faf87babfc57f58549038a77e8a6527585 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b784a43e185aa1b9230692fde621c244ef7c50bcd3e949d1ed1f03a5f2233ec0 +size 1567947 diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f69330b84a5e774ed16d949950c977bc12f40640 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ab2150c981107a3ff2e2f4528cdca9b37326f11a936bffb4fd024e86bebd02 +size 1597568 diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a31b0d592c66a50a5a42665c5fd674870a80393b --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb5c8e77d3595ea5b2040755420ab4567057c833edbc27da6ec7c1b21056b47 +size 1562396 diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a68e8e1295080f40c9794e362f2d31f23415790e --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736221235.634326]: resetting environment +[INFO] [1736221235.640118]: cleared current plan +[INFO] [1736221235.641089]: moving to home +[INFO] [1736221236.643222]: reached home +[INFO] [1736221236.644483]: reset action history +[INFO] [1736221239.355542]: environment reset +Starting episode 0 +[INFO] [1736221239.361798]: resetting environment +[INFO] [1736221239.369320]: cleared current plan +[INFO] [1736221239.370133]: moving to home +[INFO] [1736221240.372330]: reached home +[INFO] [1736221240.373691]: reset action history +[INFO] [1736221243.083222]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736221245.600954]: resetting environment +[INFO] [1736221245.605741]: cleared current plan +[INFO] [1736221245.606656]: moving to home +[INFO] [1736221246.608985]: reached home +[INFO] [1736221246.610245]: reset action history +[INFO] [1736221249.320463]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/episode_rosbags/episode_0_2025-01-06-22-40-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2128589.log +started roslaunch server http://158.130.50.37:39421/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2128893] +process[print_text-2]: started with pid [2128917] +process[rosbag_record-3]: started with pid [2128918] +started bagging! +For topic gripper_width: timestamp difference is 102571493 for nearest: 1736221259057412868 - target: 1736221258954841375 at idx 131 +For topic gripper_width: timestamp difference is 103201479 for nearest: 1736221264257361263 - target: 1736221264154159784 at idx 202 +For topic gripper_width: timestamp difference is 106641621 for nearest: 1736221268457387061 - target: 1736221268350745440 at idx 249 +For topic gripper_width: timestamp difference is 108773455 for nearest: 1736221271057426199 - target: 1736221270948652744 at idx 249 +For topic gripper_width: timestamp difference is 109751819 for nearest: 1736221272357384084 - target: 1736221272247632265 at idx 249 +For topic gripper_width: timestamp difference is 111845812 for nearest: 1736221274957438549 - target: 1736221274845592737 at idx 249 +[INFO] [1736221284.524432]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736221285.276730]: Stored episode 1. +[INFO] [1736221285.277066]: Saving video... +[INFO] [1736221285.530647]: Video saved! +Starting episode 1 +[INFO] [1736221285.544521]: resetting environment +[INFO] [1736221285.556830]: cleared current plan +[INFO] [1736221285.563033]: moving to home +[INFO] [1736221289.072152]: reached home +[INFO] [1736221289.073436]: reset action history +[INFO] [1736221291.785069]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736221303.082915]: resetting environment +[INFO] [1736221303.091439]: cleared current plan +[INFO] [1736221303.091632]: moving to home +[INFO] [1736221304.093046]: reached home +[INFO] [1736221304.093525]: reset action history +[INFO] [1736221306.809521]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/episode_rosbags/episode_1_2025-01-06-22-41-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2128589.log +started roslaunch server http://158.130.50.37:46477/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2129065] +process[print_text-5]: started with pid [2129066] +process[rosbag_record-6]: started with pid [2129067] +started bagging! +For topic gripper_width: timestamp difference is -104182345 for nearest: 1736221337390720504 - target: 1736221337494902849 at idx 249 +For topic gripper_width: timestamp difference is -102483298 for nearest: 1736221339490722154 - target: 1736221339593205452 at idx 249 +[INFO] [1736221342.305979]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736221343.094129]: Stored episode 2. +[INFO] [1736221343.098732]: Saving video... +[INFO] [1736221343.347562]: Video saved! +Starting episode 2 +[INFO] [1736221343.390375]: resetting environment +[INFO] [1736221343.413595]: cleared current plan +[INFO] [1736221343.413838]: moving to home +[INFO] [1736221346.916369]: reached home +[INFO] [1736221346.920446]: reset action history +[INFO] [1736221349.701483]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736221397.386508]: resetting environment +[INFO] [1736221397.403426]: cleared current plan +[INFO] [1736221397.403686]: moving to home +[INFO] [1736221398.405401]: reached home +[INFO] [1736221398.407941]: reset action history +[INFO] [1736221401.136410]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/episode_rosbags/episode_2_2025-01-06-22-43-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2128589.log +started roslaunch server http://158.130.50.37:42609/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2129274] +process[print_text-8]: started with pid [2129275] +process[rosbag_record-9]: started with pid [2129276] +started bagging! +[INFO] [1736221437.373150]: Storing episode... +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[INFO] [1736221438.165505]: Stored episode 3. +[INFO] [1736221438.169723]: Saving video... +[INFO] [1736221438.429595]: Video saved! +Starting episode 3 +[INFO] [1736221438.466124]: resetting environment +[INFO] [1736221438.482435]: cleared current plan +[INFO] [1736221438.482702]: moving to home +[INFO] [1736221441.687615]: reached home +[INFO] [1736221441.693547]: reset action history +[INFO] [1736221444.624632]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736221460.114504]: resetting environment +[INFO] [1736221460.144707]: cleared current plan +[INFO] [1736221460.146743]: moving to home +[INFO] [1736221461.154873]: reached home +[INFO] [1736221461.155759]: reset action history +[INFO] [1736221463.935620]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/episode_rosbags/episode_3_2025-01-06-22-44-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2128589.log +started roslaunch server http://158.130.50.37:34897/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2129419] +process[print_text-11]: started with pid [2129420] +process[rosbag_record-12]: started with pid [2129421] +started bagging! +For topic gripper_width: timestamp difference is 173939253 for nearest: 1736221477490718866 - target: 1736221477316779613 at idx 182 +For topic gripper_width: timestamp difference is 114408319 for nearest: 1736221480890727346 - target: 1736221480776319027 at idx 226 +[INFO] [1736221500.480161]: Storing episode... +[rosbag_record-12] killing on exit[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736221501.263967]: Stored episode 4. +[INFO] [1736221501.265293]: Saving video... +[INFO] [1736221501.475163]: Video saved! +Starting episode 4 +[INFO] [1736221501.538485]: resetting environment +[INFO] [1736221501.559652]: cleared current plan +[INFO] [1736221501.560498]: moving to home +[INFO] [1736221504.867044]: reached home +[INFO] [1736221504.872020]: reset action history +[INFO] [1736221507.659413]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736221512.958186]: resetting environment +[INFO] [1736221512.997310]: cleared current plan +[INFO] [1736221512.998797]: moving to home +[INFO] [1736221514.001536]: reached home +[INFO] [1736221514.008149]: reset action history +[INFO] [1736221516.801852]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/episode_rosbags/episode_4_2025-01-06-22-45-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2128589.log +started roslaunch server http://158.130.50.37:46369/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2129561] +process[print_text-14]: started with pid [2129562] +process[rosbag_record-15]: started with pid [2129563] +started bagging! +For topic gripper_width: timestamp difference is 118161047 for nearest: 1736221538957383001 - target: 1736221538839221954 at idx 249 +For topic EE_pose: timestamp difference is 129700277 for nearest: 1736221539800418708 - target: 1736221539670718431 at idx 0 +For topic EE_pose: timestamp difference is 271689938 for nearest: 1736221539942408369 - target: 1736221539670718431 at idx 0 +For topic EE_pose: timestamp difference is 306672982 for nearest: 1736221539977391413 - target: 1736221539670718431 at idx 0 +For topic EE_pose: timestamp difference is 364709419 for nearest: 1736221540035427850 - target: 1736221539670718431 at idx 0 +For topic EE_pose: timestamp difference is 566690314 for nearest: 1736221540237408745 - target: 1736221539670718431 at idx 0 +[WARN] [1736221541.071144]: Plan exhausted +[WARN] [1736221541.111830]: Plan exhausted +[INFO] [1736221553.289124]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736221554.104172]: Stored episode 5. +[INFO] [1736221554.120968]: Saving video... +[INFO] [1736221554.353083]: Video saved! +[INFO] [1736221554.423810]: resetting environment +[INFO] [1736221554.476681]: cleared current plan +[INFO] [1736221554.483995]: moving to home +[INFO] [1736221558.099268]: reached home +[INFO] [1736221558.103592]: reset action history +[INFO] [1736221560.904098]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-metadata.json b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bac04095351bdb781075303e020ea9d26c7f4f95 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T03:40:22.190152", + "startedAt": "2025-01-07T03:40:21.742569", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3747.344375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.012, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.963, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 602.1890144348145 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..954842a311ac7a5b19513f505820c951abbc607c --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "f9451f95a8b57a292306a7c34fc4be83de83308d1b1c9ced5c4f379a10cac6d6", "size": 1556320, "path": "media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4"}, "global_step": 665, "_timestamp": 1736221563.5897865, "_runtime": 341.8361954689026, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "f87c6c3fd8435e0a2b8e7945b3446fdc15074445638cc05f150d2457751fb783", "size": 1546640, "path": "media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4"}, "eval/num_success": 2.0, "episode": 25.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.09420213103294373, "eval/success_rate_ub_0.01": 0.8653537631034851, "eval/success_rate_lb_0.025": 0.08973243832588196, "eval/success_rate_ub_0.025": 0.8341690897941589, "eval/success_rate_lb_0.05": 0.12645964324474335, "eval/success_rate_ub_0.05": 0.7050612568855286, "eval/success_rate_lb_0.1": 0.11327370256185532, "eval/success_rate_ub_0.1": 0.6491125226020813, "eval/2_eval": {"_type": "video-file", "sha256": "b784a43e185aa1b9230692fde621c244ef7c50bcd3e949d1ed1f03a5f2233ec0", "size": 1567947, "path": "media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "44ab2150c981107a3ff2e2f4528cdca9b37326f11a936bffb4fd024e86bebd02", "size": 1597568, "path": "media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "cfb5c8e77d3595ea5b2040755420ab4567057c833edbc27da6ec7c1b21056b47", "size": 1562396, "path": "media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "152e8da8630452f2ed70d80bc2edc298c7b89d62a8ccf7c81614e14ed3360e9c", "size": 402, "artifact_path": "wandb-client-artifact://uhiae06avb6aqog0ego21n4ief50ww1fr9570yn88y92f5z009ml3hv6hbztfkzbdmycn8u26ykv02gmdh9odn9h2c2403sqqw7ywjij77g737te165xqws1szogqcfs/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://1ejjptxigda3xig2cvux9myvjpo0dxxb3oc40ewv89is6q5zz0eob191su0s36ce7voqggyj6bk1qwy5sfvxks2vxl3kjw2mv9qamoh3tzx0m3c64yqbhyalto927eqh:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 22, "total_num_episodes": 25, "total_success_rate": 0.88, "total_success_rate_lb_0.01": 0.7029817103933635, "total_success_rate_ub_0.01": 0.9734671934492175, "total_success_rate_lb_0.025": 0.6976236402593188, "total_success_rate_ub_0.025": 0.9621387421160531, "total_success_rate_lb_0.05": 0.7217742651257426, "total_success_rate_ub_0.05": 0.9598798117611944, "total_success_rate_lb_0.1": 0.7949399984783139, "total_success_rate_ub_0.1": 0.9499553627365946, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "5b1710e6ff2307150c3cc2fc63ccf665b8ee7925515cee52dc2c7b87522ba422", "size": 506, "artifact_path": "wandb-client-artifact://0g86s9je0a8g2e97e4p3wczg3cupo54opj42b2mq7u5rsw5shaq6cm71qz5m0ipco56dqvam85vdlrci2syd9vij57338yq3vgcit6uzntd03zx0cv9aaxzikx03dg2i/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zzuus6zunrb71czqafb7l046q1f4fbgd8lqq7clofmuu45eowebs8alos1r2apyfpq6tp4j32sdkexozqkdteqi96l7ad1ufciujhhlkyxqnc3fngekzpnxrakoddlrf:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 343}, "eval/total_success_rate": 0.8799999952316284, "eval/total_success_rate_lb_0.01": 0.70298171043396, "eval/total_success_rate_ub_0.01": 0.9734671711921692, "eval/total_success_rate_lb_0.025": 0.697623610496521, "eval/total_success_rate_ub_0.025": 0.9621387124061584, "eval/total_success_rate_lb_0.05": 0.7217742800712585, "eval/total_success_rate_ub_0.05": 0.9598798155784607, "eval/total_success_rate_lb_0.1": 0.7949399948120117, "eval/total_success_rate_ub_0.1": 0.9499553442001343} \ No newline at end of file diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug-internal.log b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d7dfdf3a264c2c0aa0b393c4b8a5d4f666fe770d --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug-internal.log @@ -0,0 +1,883 @@ +2025-01-06 22:40:21,753 INFO StreamThr :2128784 [internal.py:wandb_internal():85] W&B internal server running at pid: 2128784, started at: 2025-01-06 22:40:21.752826 +2025-01-06 22:40:21,753 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:40:21,766 INFO WriterThread:2128784 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/run-k9qvsp3e.wandb +2025-01-06 22:40:21,766 DEBUG SenderThread:2128784 [sender.py:send():379] send: header +2025-01-06 22:40:21,774 DEBUG SenderThread:2128784 [sender.py:send():379] send: run +2025-01-06 22:40:22,079 INFO SenderThread:2128784 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files +2025-01-06 22:40:22,079 INFO SenderThread:2128784 [sender.py:_start_run_threads():1188] run started: k9qvsp3e with start time 1736221221.753591 +2025-01-06 22:40:22,098 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:40:22,098 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:40:22,172 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:40:22,176 DEBUG HandlerThread:2128784 [system_info.py:__init__():26] System info init +2025-01-06 22:40:22,176 DEBUG HandlerThread:2128784 [system_info.py:__init__():41] System info init done +2025-01-06 22:40:22,176 INFO HandlerThread:2128784 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:40:22,176 INFO SystemMonitor:2128784 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:40:22,177 INFO HandlerThread:2128784 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:40:22,177 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:40:22,177 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:40:22,178 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:40:22,179 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:40:22,179 INFO SystemMonitor:2128784 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:40:22,190 DEBUG HandlerThread:2128784 [system_info.py:probe():152] Probing system +2025-01-06 22:40:22,191 DEBUG HandlerThread:2128784 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_info.py:probe():200] Probing system done +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:40:22.190152', 'startedAt': '2025-01-07T03:40:21.742569', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.344375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.012, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 602.1890144348145}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:40:22,196 INFO HandlerThread:2128784 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:40:22,196 INFO HandlerThread:2128784 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:40:22,196 DEBUG HandlerThread:2128784 [system_info.py:_save_code():44] Saving code +2025-01-06 22:40:22,197 DEBUG HandlerThread:2128784 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:40:22,197 DEBUG HandlerThread:2128784 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:40:22,260 DEBUG HandlerThread:2128784 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:40:22,260 INFO HandlerThread:2128784 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:40:22,263 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:40:22,264 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:40:22,264 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:40:22,264 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:40:22,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:40:22,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:40:22,507 DEBUG SenderThread:2128784 [sender.py:send():379] send: telemetry +2025-01-06 22:40:22,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:22,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:40:22,509 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:40:22,522 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/bla36jn6-wandb-metadata.json +2025-01-06 22:40:22,526 INFO wandb-upload_2:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/qiopqikn-diff.patch +2025-01-06 22:40:22,553 INFO HandlerThread:2128784 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/tb" + save: true +} +control { + relay_id: "7bd0dea66d0146f086ecefe898319b5c" +} +_info { + stream_id: "k9qvsp3e" +} + +2025-01-06 22:40:22,553 DEBUG HandlerThread:2128784 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:40:22,582 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/01kaoati-code/FISH/eval_robot.py +2025-01-06 22:40:22,603 DEBUG SenderThread:2128784 [sender.py:send():379] send: tbrecord +2025-01-06 22:40:22,604 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:40:22,604 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 with policy live +2025-01-06 22:40:22,796 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/s4cv1hl3-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH/eval_robot.py +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/diff.patch +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-metadata.json +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:40:23,081 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH +2025-01-06 22:40:23,082 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code +2025-01-06 22:40:23,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:24,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:25,080 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:25,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:26,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:27,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:27,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:28,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:29,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:30,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:31,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:32,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:33,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:33,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:34,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:35,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:35,633 DEBUG SenderThread:2128784 [sender.py:send():379] send: config +2025-01-06 22:40:36,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:37,083 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:37,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:40:37,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:40:37,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:38,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:38,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:39,084 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:39,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:40,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:41,084 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:41,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:42,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:43,085 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:43,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:44,125 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:44,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:45,086 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:45,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:46,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:47,086 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:47,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:48,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:49,087 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:49,320 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:49,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:50,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:51,088 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:40:51,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:52,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:40:52,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:40:52,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:53,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:54,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:54,584 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:40:55,089 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml +2025-01-06 22:40:55,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:56,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:57,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:58,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:40:59,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:00,105 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:00,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:01,090 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:01,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:02,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:03,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:04,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:05,305 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:05,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:06,512 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:07,092 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:07,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:07,506 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:07,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:08,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:09,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:10,509 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:10,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:11,093 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:11,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:12,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:13,093 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:13,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:14,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:15,094 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:15,513 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:16,009 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:16,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:17,095 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:17,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:18,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:19,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:20,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:21,508 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:21,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:22,180 DEBUG SystemMonitor:2128784 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:41:22,180 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:41:22,506 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:22,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:22,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:23,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:24,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:25,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:25,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:41:25,573 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:41:25,574 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 with policy now +2025-01-06 22:41:25,577 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:41:25,577 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:41:25,577 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:41:25,839 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/bz6t9kbj-media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:41:26,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:41:26,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:41:26,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media +2025-01-06 22:41:26,098 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:41:26,098 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos +2025-01-06 22:41:26,374 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/j41ugirl-media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:41:26,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:26,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:27,097 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:27,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:28,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:29,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:30,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:31,099 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:31,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:31,785 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:32,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:33,100 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:33,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:34,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:35,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:36,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:37,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:37,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:37,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:37,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:38,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:39,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:40,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:41,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:42,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:43,072 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:43,103 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:41:43,340 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/kt4aldiq-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:41:43,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:44,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:45,103 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:45,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:46,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:47,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:48,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:48,664 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:49,105 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:41:49,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:50,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:51,520 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:52,185 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:41:52,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:41:52,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:41:52,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:53,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:54,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:54,613 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:41:55,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:56,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:57,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:58,523 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:41:59,522 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:00,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:00,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:01,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:02,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:03,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:04,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:05,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:05,526 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:06,526 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:07,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:07,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:07,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:08,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:09,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:10,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:10,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:11,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:12,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:13,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:14,527 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:15,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:16,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:16,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:17,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:18,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:19,114 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:19,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:20,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:21,115 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:21,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:21,729 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:22,181 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:42:22,507 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:22,507 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:22,531 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:23,384 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:42:23,385 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 with policy now +2025-01-06 22:42:23,387 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:42:23,388 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:42:23,388 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:42:23,388 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:42:23,530 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:23,646 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/tvpuqpr1-media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:42:24,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:42:24,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:42:24,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:42:24,453 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/jw837zy1-media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:42:24,533 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:25,116 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:25,533 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:26,545 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:26,917 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:27,544 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:28,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:29,118 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:29,547 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:30,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:31,118 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:42:31,539 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:32,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:32,540 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:33,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:34,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:35,545 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:36,548 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:37,515 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:37,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:37,518 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:37,561 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:38,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:39,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:40,549 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:41,554 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:42,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:42,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:43,553 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:44,552 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:45,556 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:46,551 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:47,552 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:48,518 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:48,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:49,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:50,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:51,556 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:52,182 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:42:52,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:42:52,515 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:42:52,558 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:53,559 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:53,596 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:54,560 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:55,561 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:56,562 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:57,562 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:58,563 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:42:59,521 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:42:59,570 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:00,563 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:01,570 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:02,567 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:03,567 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:04,529 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:04,569 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:05,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:06,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:07,511 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:07,511 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:07,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:08,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:09,573 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:09,638 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:10,572 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:11,576 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:12,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:13,574 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:14,576 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:15,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:15,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:16,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:17,577 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:18,038 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:43:18,038 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:43:18,039 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:43:18,133 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:43:18,134 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:43:18,369 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/7v08gmdz-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:43:18,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:19,133 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:19,580 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:20,588 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:21,134 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:21,141 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:21,584 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:22,182 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:43:22,510 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:22,510 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:22,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:23,135 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:23,581 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:24,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:25,136 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:25,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:26,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:27,073 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:27,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:28,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:29,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:30,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:31,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:32,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:32,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:33,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:34,585 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:35,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:36,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:37,514 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:37,514 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:37,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:37,597 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:38,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:39,588 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:40,586 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:41,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:42,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:43,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:43,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:44,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:45,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:46,587 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:47,591 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:48,524 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:48,588 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:49,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:50,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:51,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:52,183 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:43:52,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:43:52,518 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:43:52,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:53,579 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:53,591 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:54,590 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:55,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:56,589 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:57,590 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:58,460 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:43:58,460 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 with policy now +2025-01-06 22:43:58,465 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:43:58,466 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:43:58,466 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:43:58,466 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:43:58,590 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:43:58,720 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/g1wi6unv-media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:43:59,147 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:43:59,147 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:43:59,148 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:43:59,148 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:43:59,450 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/wezinnen-media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:43:59,483 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:43:59,595 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:00,596 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:01,148 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:01,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:02,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:03,149 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:03,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:04,536 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:04,605 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:05,601 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:06,600 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:07,150 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:07,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:07,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:07,611 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:08,602 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:09,599 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:09,607 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:10,608 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:11,609 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:12,612 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:13,605 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:14,610 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:15,538 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:15,612 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:16,612 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:17,614 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:18,615 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:19,616 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:20,154 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:44:20,398 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/awyasg86-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:44:20,619 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:20,890 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:44:20,891 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:44:20,891 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:20,891 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:44:21,154 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:44:21,155 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:21,619 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:22,183 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:44:22,522 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:22,522 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:22,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:23,155 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:23,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:24,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:25,156 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:25,619 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:25,902 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:26,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:27,157 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:27,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:28,620 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:29,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:30,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:31,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:31,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:32,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:33,621 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:34,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:35,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:36,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:36,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:37,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:37,517 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:37,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:38,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:39,161 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:39,622 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:40,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:41,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:41,937 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:42,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:43,162 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:44:43,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:44,623 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:45,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:46,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:47,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:47,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:48,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:49,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:50,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:51,624 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:52,184 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:44:52,517 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:44:52,518 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:44:52,578 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:52,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:53,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:54,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:55,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:56,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:57,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:58,535 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:44:58,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:44:59,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:00,625 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:01,522 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:45:01,522 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 with policy now +2025-01-06 22:45:01,525 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:45:01,526 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:45:01,526 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:45:01,526 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:45:01,626 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:01,787 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/48s1juc4-media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:45:02,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:45:02,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:45:02,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:45:02,467 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/tm6tn434-media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:45:02,635 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:03,169 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:03,542 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:03,630 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:04,631 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:05,641 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:06,643 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:07,170 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:07,522 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:07,522 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:07,643 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:08,647 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:08,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:09,171 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:09,654 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:10,655 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:11,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:12,656 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:12,953 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:45:12,954 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:45:12,955 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:45:13,172 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:45:13,173 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:45:13,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:13,999 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:14,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:15,173 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:15,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:16,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:17,665 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:18,656 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:19,174 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:19,353 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:19,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:20,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:21,175 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:21,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:22,184 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:45:22,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:22,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:22,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:23,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:24,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:24,665 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:25,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:26,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:27,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:28,657 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:29,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:30,541 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:30,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:31,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:32,658 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:33,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:34,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:35,541 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:35,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:36,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:37,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:37,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:37,660 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:38,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:39,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:40,635 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:40,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:41,182 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:41,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:42,661 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:43,183 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:43,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:44,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:45,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:46,541 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:46,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:47,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:48,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:49,663 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:50,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:51,545 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:51,662 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:52,184 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:45:52,519 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:45:52,519 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:45:52,663 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:53,667 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:54,418 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:45:54,418 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 with policy now +2025-01-06 22:45:54,423 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:45:54,423 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:45:54,423 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:45:54,434 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:45:54,665 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:54,721 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/9nb25diq-media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:45:55,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval +2025-01-06 22:45:55,471 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/19zishh9-media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:45:55,674 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:56,676 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:57,187 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:57,488 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:45:57,681 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:58,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:45:59,188 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:45:59,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:00,680 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:01,682 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:02,555 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:46:02,684 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:03,016 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:03,016 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,017 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,118 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:46:03,119 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table +2025-01-06 22:46:03,190 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval +2025-01-06 22:46:03,301 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:46:03,392 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/72s13plb-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:03,432 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/2k8grycg-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:03,607 INFO wandb-upload_2:2128784 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpm0rttqun +2025-01-06 22:46:03,677 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:03,701 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:46:03,745 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:46:03,988 INFO SenderThread:2128784 [sender.py:send_request_log_artifact():1525] logged artifact run-k9qvsp3e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTQxNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2MDI0NQ==', 'latestArtifact': None}} +2025-01-06 22:46:03,988 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:46:03,988 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json with policy now +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,989 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send():379] send: telemetry +2025-01-06 22:46:03,989 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,990 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,990 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,990 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,990 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,992 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,993 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,993 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,993 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,993 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,994 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,994 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,994 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,994 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,995 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,995 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,995 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,995 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,995 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,996 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,996 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:03,996 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:03,996 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:04,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval +2025-01-06 22:46:04,234 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/x7qt8ify-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:04,369 INFO wandb-upload_2:2128784 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk36k0l6r +2025-01-06 22:46:04,382 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/ggch35pc-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:04,677 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:04,780 INFO SenderThread:2128784 [sender.py:send_request_log_artifact():1525] logged artifact run-k9qvsp3e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTQ0MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc2MDI0Ng==', 'latestArtifact': None}} +2025-01-06 22:46:04,780 DEBUG SenderThread:2128784 [sender.py:send():379] send: files +2025-01-06 22:46:04,780 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json with policy now +2025-01-06 22:46:04,780 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:04,781 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:04,781 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:05,029 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/4up7fth4-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:05,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:05,191 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:05,677 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:05,805 DEBUG SenderThread:2128784 [sender.py:send():379] send: exit +2025-01-06 22:46:05,805 INFO SenderThread:2128784 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:send_exit():588] handling runtime: 343 +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:send_exit():594] send defer +2025-01-06 22:46:05,806 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,806 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:46:05,806 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:46:05,806 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:46:05,806 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,806 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:46:05,807 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,807 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:46:05,807 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:46:05,807 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,807 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:46:05,807 INFO HandlerThread:2128784 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:46:05,807 DEBUG SystemMonitor:2128784 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:46:05,807 DEBUG SystemMonitor:2128784 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:46:05,808 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:46:05,808 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:46:05,859 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:46:05,859 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:46:05,859 INFO HandlerThread:2128784 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:46:05,860 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:46:05,860 DEBUG SenderThread:2128784 [sender.py:send():379] send: stats +2025-01-06 22:46:05,860 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,860 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:46:05,860 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:46:05,860 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:46:05,860 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:05,861 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:46:06,192 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:07,192 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:12,022 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:12,022 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:46:12,022 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:46:12,024 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,024 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:46:12,024 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:46:12,024 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:12,024 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:12,024 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:46:12,024 DEBUG SenderThread:2128784 [sender.py:send():379] send: history +2025-01-06 22:46:12,025 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:46:12,025 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:12,025 DEBUG SenderThread:2128784 [sender.py:send():379] send: summary +2025-01-06 22:46:12,029 INFO SenderThread:2128784 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:46:12,029 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,029 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:46:12,029 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:46:12,029 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:12,029 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:46:12,030 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,030 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:46:12,033 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:46:12,160 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:46:12,160 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:12,160 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:46:12,161 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:12,161 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:46:12,194 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:12,195 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml +2025-01-06 22:46:12,807 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:13,102 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:46:13,102 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:13,102 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:13,102 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:46:13,102 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:13,102 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:46:13,102 INFO SenderThread:2128784 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:46:13,105 INFO SenderThread:2128784 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:46:13,107 INFO SenderThread:2128784 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:46:13,122 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:46:13,122 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:13,122 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:46:13,123 DEBUG SenderThread:2128784 [sender.py:send():379] send: artifact +2025-01-06 22:46:13,195 INFO Thread-12 :2128784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:13,493 INFO wandb-upload_1:2128784 [upload_job.py:push():85] Skipped uploading /tmp/tmp_60dcf4r/wandb-job.json +2025-01-06 22:46:13,493 INFO wandb-upload_2:2128784 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuv317_7z +2025-01-06 22:46:13,612 INFO wandb-upload_0:2128784 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgpe_9mkz +2025-01-06 22:46:13,807 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:14,307 INFO SenderThread:2128784 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjYzMTc3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'versionIndex': 428}}} +2025-01-06 22:46:14,307 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:14,307 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:46:14,307 INFO SenderThread:2128784 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml config.yaml +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json wandb-summary.json +2025-01-06 22:46:15,196 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt requirements.txt +2025-01-06 22:46:15,197 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:46:15,199 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:15,200 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log output.log +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/diff.patch diff.patch +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 media/videos/eval/4_eval_4_cfb5c8e77d3595ea5b20.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 media/videos/eval/1_eval_1_f87c6c3fd8435e0a2b8e.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 media/videos/eval/3_eval_3_44ab2150c981107a3ff2.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 media/videos/eval/2_eval_2_b784a43e185aa1b92306.mp4 +2025-01-06 22:46:15,202 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 media/videos/eval/0_eval_0_f9451f95a8b57a292306.mp4 +2025-01-06 22:46:15,203 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-06 22:46:15,203 INFO SenderThread:2128784 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-06 22:46:15,203 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:46:15,203 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:15,206 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,206 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:46:15,206 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,206 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:46:15,206 INFO SenderThread:2128784 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:46:15,369 INFO wandb-upload_2:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/wandb-summary.json +2025-01-06 22:46:15,386 INFO wandb-upload_1:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/config.yaml +2025-01-06 22:46:15,386 INFO wandb-upload_0:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/requirements.txt +2025-01-06 22:46:15,456 INFO wandb-upload_4:2128784 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/files/output.log +2025-01-06 22:46:15,490 INFO wandb-upload_3:2128784 [upload_job.py:push():130] Uploaded file /tmp/tmphgppz65kwandb/5ac8u05j-events.out.tfevents.1736221222.leonmkim-ROG-Strix-G15CS-G15CS.2128589.0 +2025-01-06 22:46:15,691 INFO Thread-11 (_thread_body):2128784 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:46:15,691 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,691 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:46:15,691 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,691 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:46:15,692 INFO SenderThread:2128784 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:46:15,694 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:46:15,694 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,695 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:46:15,695 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,695 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:46:15,695 INFO SenderThread:2128784 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:46:15,750 INFO SenderThread:2128784 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:46:15,751 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:46:15,751 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,751 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:46:15,751 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,751 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:46:15,751 INFO SenderThread:2128784 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:46:15,751 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:46:15,751 INFO HandlerThread:2128784 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send():379] send: final +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send():379] send: footer +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: defer +2025-01-06 22:46:15,752 INFO SenderThread:2128784 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:46:15,752 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:15,752 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:15,753 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:46:15,753 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:46:15,753 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:46:15,754 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:46:15,755 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:46:15,755 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:46:15,807 DEBUG SenderThread:2128784 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:46:15,807 INFO MainThread:2128784 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:46:15,808 INFO MainThread:2128784 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:46:15,808 INFO MainThread:2128784 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:46:15,808 DEBUG HandlerThread:2128784 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:46:15,808 INFO HandlerThread:2128784 [handler.py:finish():882] shutting down handler +2025-01-06 22:46:16,753 INFO WriterThread:2128784 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/run-k9qvsp3e.wandb +2025-01-06 22:46:16,807 INFO SenderThread:2128784 [sender.py:finish():1615] shutting down sender +2025-01-06 22:46:16,807 INFO SenderThread:2128784 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:46:16,807 INFO SenderThread:2128784 [file_pusher.py:join():175] waiting for file pusher diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug.log b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..da1e0e18578083c6e9497bd1736c9a1c46fab0b0 --- /dev/null +++ b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Configure stats pid to 2128589 +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/settings +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:40:21,744 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug.log +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/logs/debug-internal.log +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:40:21,746 INFO MainThread:2128589 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e'} +2025-01-06 22:40:21,747 INFO MainThread:2128589 [wandb_init.py:init():619] starting backend +2025-01-06 22:40:21,747 INFO MainThread:2128589 [wandb_init.py:init():623] setting up manager +2025-01-06 22:40:21,752 INFO MainThread:2128589 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:40:21,752 INFO MainThread:2128589 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:40:21,765 INFO MainThread:2128589 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:40:21,772 INFO MainThread:2128589 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:40:22,097 INFO MainThread:2128589 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:40:22,166 INFO MainThread:2128589 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:40:22,166 INFO MainThread:2128589 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:40:22,506 INFO MainThread:2128589 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:40:22,509 INFO MainThread:2128589 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:40:22,509 INFO MainThread:2128589 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/k9qvsp3e/tb, True +2025-01-06 22:40:35,633 INFO MainThread:2128589 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:46:16,809 WARNING MsgRouterThr:2128589 [router.py:message_loop():77] message_loop has been closed diff --git a/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/run-k9qvsp3e.wandb b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/run-k9qvsp3e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ffd2e6fc1f5bfe01c761cc7c7e5284466fe49ef4 Binary files /dev/null and b/k9qvsp3e/wandb/run-20250106_224021-k9qvsp3e/run-k9qvsp3e.wandb differ diff --git a/oagunwyg/.hydra/config.yaml b/oagunwyg/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/oagunwyg/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/oagunwyg/.hydra/hydra.yaml b/oagunwyg/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..492f4f29f8648621704ab8edb0597ef774a7433d --- /dev/null +++ b/oagunwyg/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/oagunwyg/.hydra/overrides.yaml b/oagunwyg/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/oagunwyg/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/oagunwyg/episode_rosbags/color_K.npy b/oagunwyg/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/oagunwyg/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/oagunwyg/episode_rosbags/episode_0_2025-01-06-20-16-24.bag b/oagunwyg/episode_rosbags/episode_0_2025-01-06-20-16-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..869747199bf642ee9d651c8e2cac0877895ff9ae --- /dev/null +++ b/oagunwyg/episode_rosbags/episode_0_2025-01-06-20-16-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ffbf84d26fdb26a2c071555412b67a05691fe3a81ee4fa7bc02b9dbbf3c33d6 +size 2609104370 diff --git a/oagunwyg/episode_rosbags/episode_1_2025-01-06-20-17-12.bag b/oagunwyg/episode_rosbags/episode_1_2025-01-06-20-17-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..9a37a84d035a6bb3d08929b255ee6fbe56a54922 --- /dev/null +++ b/oagunwyg/episode_rosbags/episode_1_2025-01-06-20-17-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:408441bf390b392f4bfb419b0c24237f2194f7c39b57bb711dffca627c01debc +size 2530633944 diff --git a/oagunwyg/episode_rosbags/episode_2_2025-01-06-20-17-59.bag b/oagunwyg/episode_rosbags/episode_2_2025-01-06-20-17-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..f1e52b6a39911a69e351878de0b4e22006a2fa79 --- /dev/null +++ b/oagunwyg/episode_rosbags/episode_2_2025-01-06-20-17-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dacaece021c25b23616f7a045540edda7eac5f396dbd65149c6b7b40a986c34a +size 2605521066 diff --git a/oagunwyg/episode_rosbags/episode_3_2025-01-06-20-18-55.bag b/oagunwyg/episode_rosbags/episode_3_2025-01-06-20-18-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..c20c4143a3f478db6ee2e4e0e014fa39e6b80886 --- /dev/null +++ b/oagunwyg/episode_rosbags/episode_3_2025-01-06-20-18-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bf28004c997f43cb948b4550804543de349b9053f70915e282e300c4fd99c80 +size 2609866967 diff --git a/oagunwyg/eval_video/0_eval.mp4 b/oagunwyg/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..099b9a8e43ab564deccf3e642bb5ec5173b03257 --- /dev/null +++ b/oagunwyg/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77806420d75eb9ff49fc3a34e3fedb731f74f8ef2f5e25ca46e4ae0df0fdd3ce +size 1432277 diff --git a/oagunwyg/eval_video/1_eval.mp4 b/oagunwyg/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a40be3bbb2d71134717ab95441ce41306fb5adb --- /dev/null +++ b/oagunwyg/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6260f5c906394d6a4f4979b7c8f81ff6cc2d39809dd6a858d9690bc77a3f53 +size 1389414 diff --git a/oagunwyg/eval_video/2_eval.mp4 b/oagunwyg/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53c3f0c93073b654185b7bf3d61351d124cc95ba --- /dev/null +++ b/oagunwyg/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112b95f61f166bf491344b68561ce66a186890d9953c444be552658a09e883b0 +size 1427754 diff --git a/oagunwyg/eval_video/3_eval.mp4 b/oagunwyg/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb3d9d424d856a362e300268da167422cb49706c --- /dev/null +++ b/oagunwyg/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0f62edae868114b9186f36bb50e06fbc82b1a68bedad4e05d70995d7d077fd2 +size 1427838 diff --git a/oagunwyg/eval_video/4_eval.mp4 b/oagunwyg/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..339d3504322ba05f6b43d457d39ffcef2a685b0b --- /dev/null +++ b/oagunwyg/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e7800e9e899160f2ce3d7e9f51ec0a6b468ce999404b7fe00713cb6d021a83 +size 1470161 diff --git a/oagunwyg/tb/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 b/oagunwyg/tb/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 new file mode 100644 index 0000000000000000000000000000000000000000..b57ff37acf30b969c8212ac93cf55555004b8133 --- /dev/null +++ b/oagunwyg/tb/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e96a132dbf33289e387a822e4c111e7ece15bd511640d9fda6cfc52d43e6432 +size 6539 diff --git a/oagunwyg/wandb/debug-internal.log b/oagunwyg/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5ca4ee1ae4ef62bb4fdc16883bb2b197d457d03b --- /dev/null +++ b/oagunwyg/wandb/debug-internal.log @@ -0,0 +1,780 @@ +2025-01-06 20:16:03,517 INFO StreamThr :2106904 [internal.py:wandb_internal():85] W&B internal server running at pid: 2106904, started at: 2025-01-06 20:16:03.517246 +2025-01-06 20:16:03,518 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:16:03,531 INFO WriterThread:2106904 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/run-oagunwyg.wandb +2025-01-06 20:16:03,531 DEBUG SenderThread:2106904 [sender.py:send():379] send: header +2025-01-06 20:16:03,541 DEBUG SenderThread:2106904 [sender.py:send():379] send: run +2025-01-06 20:16:03,921 INFO SenderThread:2106904 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files +2025-01-06 20:16:03,921 INFO SenderThread:2106904 [sender.py:_start_run_threads():1188] run started: oagunwyg with start time 1736212563.517784 +2025-01-06 20:16:03,936 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:16:03,937 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:16:04,012 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:16:04,017 DEBUG HandlerThread:2106904 [system_info.py:__init__():26] System info init +2025-01-06 20:16:04,017 DEBUG HandlerThread:2106904 [system_info.py:__init__():41] System info init done +2025-01-06 20:16:04,017 INFO HandlerThread:2106904 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:16:04,017 INFO SystemMonitor:2106904 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:16:04,017 INFO HandlerThread:2106904 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:16:04,017 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:16:04,018 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:16:04,019 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:16:04,019 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:16:04,020 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:16:04,030 DEBUG HandlerThread:2106904 [system_info.py:probe():152] Probing system +2025-01-06 20:16:04,031 DEBUG HandlerThread:2106904 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_info.py:probe():200] Probing system done +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:16:04.030562', 'startedAt': '2025-01-07T01:16:03.503587', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.769375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4484.452, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 462.5564193725586}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:16:04,036 INFO HandlerThread:2106904 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:16:04,036 INFO HandlerThread:2106904 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_info.py:_save_code():44] Saving code +2025-01-06 20:16:04,038 DEBUG HandlerThread:2106904 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:16:04,038 DEBUG HandlerThread:2106904 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:16:04,099 DEBUG HandlerThread:2106904 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:16:04,100 INFO HandlerThread:2106904 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:16:04,103 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:16:04,104 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:16:04,104 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:16:04,105 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:16:04,352 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:16:04,352 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:16:04,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:04,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:04,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:04,355 INFO HandlerThread:2106904 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/tb" + save: true +} +control { + relay_id: "42b887af31504a9fb5f1f21b3fb07c55" +} +_info { + stream_id: "oagunwyg" +} + +2025-01-06 20:16:04,355 DEBUG HandlerThread:2106904 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:16:04,368 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/wbgygke7-code/FISH/eval_robot.py +2025-01-06 20:16:04,377 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/n8xz4rgy-wandb-metadata.json +2025-01-06 20:16:04,386 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/et0293sr-diff.patch +2025-01-06 20:16:04,509 DEBUG SenderThread:2106904 [sender.py:send():379] send: telemetry +2025-01-06 20:16:04,510 DEBUG SenderThread:2106904 [sender.py:send():379] send: tbrecord +2025-01-06 20:16:04,510 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:16:04,510 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 with policy live +2025-01-06 20:16:04,697 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/9i13xyj8-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/diff.patch +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-metadata.json +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH/eval_robot.py +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code +2025-01-06 20:16:04,924 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH +2025-01-06 20:16:05,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:06,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:06,922 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:07,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:08,209 DEBUG SenderThread:2106904 [sender.py:send():379] send: config +2025-01-06 20:16:08,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:09,217 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:09,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:10,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:10,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:11,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:12,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:12,924 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:13,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:14,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:14,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:14,924 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:15,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:16,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:16,925 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:17,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:18,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:19,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:19,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:19,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:19,443 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:20,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:21,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:22,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:22,927 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:23,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:24,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:24,550 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:25,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:26,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:26,928 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:27,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:28,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:29,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:30,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:30,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:31,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:32,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:33,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:34,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:34,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:34,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:35,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:35,424 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:35,930 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml +2025-01-06 20:16:36,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:37,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:38,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:39,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:40,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:41,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:41,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:42,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:43,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:44,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:45,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:46,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:46,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:47,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:48,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:49,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:49,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:49,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:50,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:51,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:51,410 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:52,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:53,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:54,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:55,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:56,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:57,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:57,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:58,361 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:59,361 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:00,363 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:00,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:17:00,450 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:17:00,450 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 with policy now +2025-01-06 20:17:00,450 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:17:00,450 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:17:00,451 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:17:00,690 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/kov16fo0-media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media +2025-01-06 20:17:01,204 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/lqok3r61-media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:17:01,363 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:02,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:02,452 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:02,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:03,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:04,021 DEBUG SystemMonitor:2106904 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:17:04,021 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:17:04,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:04,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:04,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:05,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:06,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:06,939 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:07,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:07,937 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:08,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:08,939 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:08,940 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:17:09,180 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/adrkwjol-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:17:09,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:10,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:10,940 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:11,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:12,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:13,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:13,624 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:14,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:14,942 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:15,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:16,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:16,942 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:17,366 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:18,366 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:18,968 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:19,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:19,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:19,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:20,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:21,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:22,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:23,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:24,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:24,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:25,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:26,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:27,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:28,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:29,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:29,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:30,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:31,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:32,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:33,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:34,022 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:17:34,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:34,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:34,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:34,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:35,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:36,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:37,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:38,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:39,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:40,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:40,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:41,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:42,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:43,372 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:44,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:45,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:45,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:46,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:47,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:47,669 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:17:47,669 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 with policy now +2025-01-06 20:17:47,672 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:17:47,672 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:17:47,672 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:17:47,693 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:17:47,903 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/5tunx9nf-media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:17:47,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:17:47,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:17:47,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:17:48,222 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/gkxl7vwk-media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:17:48,373 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:48,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:49,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:49,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:49,375 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:49,954 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:50,374 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:50,468 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:51,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:52,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:52,955 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:53,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:54,376 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:55,376 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:55,967 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:56,376 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:56,774 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:17:56,774 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:17:56,774 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:17:56,956 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:17:56,956 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:56,956 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:17:57,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:58,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:58,957 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:59,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:00,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:00,957 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:01,106 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:01,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:02,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:03,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:04,022 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:18:04,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:04,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:04,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:05,378 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:06,378 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:06,455 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:07,378 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:08,379 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:09,379 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:10,379 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:11,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:12,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:12,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:13,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:14,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:15,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:16,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:17,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:17,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:18,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:19,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:19,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:19,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:20,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:21,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:22,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:22,452 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:23,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:24,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:25,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:26,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:27,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:28,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:28,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:29,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:30,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:31,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:32,383 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:33,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:33,383 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:34,023 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:18:34,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:34,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:34,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:35,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:36,011 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:18:36,011 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 with policy now +2025-01-06 20:18:36,011 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:18:36,012 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:18:36,012 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:18:36,025 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:18:36,257 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/5un1at3d-media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:18:36,386 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:36,986 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:18:36,987 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:36,987 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:18:36,987 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:18:37,280 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/papyxprm-media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:18:37,391 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:38,396 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:39,027 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:39,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:40,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:40,988 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:41,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:42,395 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:43,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:44,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:44,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:44,989 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:45,395 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:46,398 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:47,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:48,395 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:49,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:49,358 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:49,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:49,503 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:50,399 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:51,396 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:51,992 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:18:52,245 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/rtcnlw8t-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:18:52,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:52,619 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:18:52,619 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:18:52,619 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:18:52,992 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:18:52,993 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:53,400 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:54,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:54,993 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:55,362 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:55,405 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:56,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:56,993 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:57,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:58,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:59,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:00,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:01,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:01,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:02,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:03,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:04,023 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:19:04,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:04,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:04,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:05,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:06,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:06,493 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:07,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:08,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:09,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:10,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:11,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:12,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:12,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:13,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:14,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:15,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:16,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:17,405 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:18,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:18,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:19,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:19,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:19,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:20,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:21,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:22,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:23,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:23,412 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:24,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:25,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:26,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:27,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:28,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:29,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:29,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:30,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:31,417 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:31,865 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:19:31,865 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 with policy now +2025-01-06 20:19:31,866 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:19:31,868 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:19:31,868 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:19:31,879 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:19:32,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:19:32,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:19:32,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:19:32,098 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/zx2xu9fj-media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:19:32,361 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/ki6766a9-media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:19:32,412 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:33,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:33,418 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:34,024 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:19:34,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:34,359 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:34,416 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:34,434 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:35,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:36,423 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:37,007 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:37,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:38,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:39,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:40,367 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:40,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:41,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:41,425 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:41,675 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:19:41,675 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:19:41,676 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:19:42,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:19:42,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:19:42,425 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:43,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:43,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:44,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:45,010 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:45,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:46,261 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:46,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:47,011 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:47,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:48,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:49,012 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:49,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:49,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:49,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:50,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:51,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:51,466 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:52,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:53,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:54,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:55,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:56,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:57,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:57,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:58,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:59,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:00,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:01,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:02,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:02,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:03,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:04,024 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:20:04,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:20:04,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:20:04,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:05,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:06,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:07,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:07,481 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:08,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:09,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:10,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:11,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:12,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:13,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:13,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:14,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:15,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:16,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:17,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:18,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:18,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:19,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:20:19,357 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:20:19,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:20,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:21,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:22,447 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:22,740 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:20:22,740 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 with policy now +2025-01-06 20:20:22,744 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:20:22,745 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:22,745 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:22,754 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:22,978 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/qhmdiuow-media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:23,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:23,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:23,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:23,025 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:20:23,303 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/2n4ohrra-media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:23,439 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:23,770 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:24,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:24,452 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:25,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:25,445 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:26,447 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:27,445 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:28,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:29,026 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:29,374 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:29,450 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:30,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:31,026 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:31,455 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:32,027 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:32,273 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/rnz4c44e-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:32,325 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:20:32,326 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:20:32,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:32,522 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:20:32,805 INFO wandb-upload_1:2106904 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpiq9wfd72 +2025-01-06 20:20:32,897 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:20:32,907 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:20:33,027 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:33,027 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table +2025-01-06 20:20:33,171 INFO SenderThread:2106904 [sender.py:send_request_log_artifact():1525] logged artifact run-oagunwyg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI0ODQ2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYxNDEzNA==', 'latestArtifact': None}} +2025-01-06 20:20:33,171 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:20:33,171 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 20:20:33,171 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:33,172 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,172 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,172 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:33,172 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,173 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,173 DEBUG SenderThread:2106904 [sender.py:send():379] send: telemetry +2025-01-06 20:20:33,173 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,173 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,173 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,174 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,174 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,174 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,174 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,174 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,175 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,175 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,175 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,175 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,176 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,176 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,176 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,176 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,177 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,177 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,177 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,177 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,178 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,178 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,178 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:20:33,351 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/ga6vwjro-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:33,373 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/9wbv3zfg-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:33,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:33,590 INFO wandb-upload_1:2106904 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1833up9y +2025-01-06 20:20:33,595 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/wwlds7kz-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:33,973 INFO SenderThread:2106904 [sender.py:send_request_log_artifact():1525] logged artifact run-oagunwyg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI0ODU1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYxNDIyMA==', 'latestArtifact': None}} +2025-01-06 20:20:33,973 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:20:33,973 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-06 20:20:33,973 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:33,973 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,974 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:34,026 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:20:34,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:34,217 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/i35iyhl9-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:34,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:20:34,357 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:20:34,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:34,469 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:35,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:35,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:36,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:37,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:38,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:39,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:40,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:40,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:41,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:42,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:43,450 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:44,450 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:44,534 DEBUG SenderThread:2106904 [sender.py:send():379] send: exit +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:send_exit():588] handling runtime: 280 +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:send_exit():594] send defer +2025-01-06 20:20:44,534 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,534 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:20:44,535 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:20:44,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,535 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:20:44,535 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:20:44,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,535 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:20:44,535 INFO HandlerThread:2106904 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:20:44,536 DEBUG SystemMonitor:2106904 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:20:44,536 DEBUG SystemMonitor:2106904 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:20:44,536 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:20:44,537 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:20:44,589 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:20:44,590 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:20:44,590 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:20:44,590 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,590 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:20:44,590 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:20:44,590 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:20:44,591 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,591 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:20:44,591 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,591 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:20:44,591 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:20:44,591 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,591 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:20:45,032 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:47,033 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:50,746 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,746 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:20:50,746 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:20:50,746 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:50,746 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:50,747 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:50,747 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:50,747 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:20:50,748 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:50,748 DEBUG SenderThread:2106904 [sender.py:send():379] send: summary +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:50,752 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:20:50,752 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:50,752 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:20:50,752 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:20:50,756 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:50,878 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:20:50,878 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:50,878 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:20:50,878 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,878 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:20:51,035 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:51,035 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml +2025-01-06 20:20:51,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:53,793 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:20:53,793 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:53,793 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:53,794 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:20:53,794 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:53,794 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:20:53,794 INFO SenderThread:2106904 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:20:53,796 INFO SenderThread:2106904 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:20:53,798 INFO SenderThread:2106904 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:20:53,808 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:20:53,808 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:53,808 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:20:53,809 DEBUG SenderThread:2106904 [sender.py:send():379] send: artifact +2025-01-06 20:20:54,026 INFO SenderThread:2106904 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk3NDE3OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:20:54,026 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,027 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:20:54,027 INFO SenderThread:2106904 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml config.yaml +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json wandb-summary.json +2025-01-06 20:20:54,037 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt requirements.txt +2025-01-06 20:20:54,039 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:20:54,040 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:54,040 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log output.log +2025-01-06 20:20:54,040 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/diff.patch diff.patch +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:20:54,043 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:54,043 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:54,043 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:20:54,044 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,044 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:20:54,046 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,046 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:20:54,046 INFO SenderThread:2106904 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:20:54,225 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt +2025-01-06 20:20:54,232 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:54,252 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml +2025-01-06 20:20:54,280 INFO wandb-upload_4:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:54,309 INFO wandb-upload_3:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/osqx1d02-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:54,509 INFO Thread-11 (_thread_body):2106904 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:20:54,509 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,509 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:20:54,510 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,510 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:20:54,510 INFO SenderThread:2106904 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:20:54,511 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:20:54,511 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,511 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:20:54,511 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,511 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:20:54,511 INFO SenderThread:2106904 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:20:54,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:20:54,714 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:54,714 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,714 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:20:54,714 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:20:54,715 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,715 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:20:54,715 DEBUG SenderThread:2106904 [sender.py:send():379] send: final +2025-01-06 20:20:54,715 DEBUG SenderThread:2106904 [sender.py:send():379] send: footer +2025-01-06 20:20:54,715 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,715 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:20:54,715 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:54,716 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:20:54,716 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:54,716 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:54,716 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:20:54,716 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:20:54,717 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:20:54,719 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:54,763 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:54,763 INFO MainThread:2106904 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:20:54,763 INFO MainThread:2106904 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:20:54,764 INFO MainThread:2106904 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:20:54,764 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:20:54,764 INFO HandlerThread:2106904 [handler.py:finish():882] shutting down handler +2025-01-06 20:20:55,716 INFO WriterThread:2106904 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/run-oagunwyg.wandb +2025-01-06 20:20:55,763 INFO SenderThread:2106904 [sender.py:finish():1615] shutting down sender +2025-01-06 20:20:55,763 INFO SenderThread:2106904 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:20:55,763 INFO SenderThread:2106904 [file_pusher.py:join():175] waiting for file pusher diff --git a/oagunwyg/wandb/debug.log b/oagunwyg/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4979ddd98971c54f7f3e93f4e07185274de46067 --- /dev/null +++ b/oagunwyg/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:16:03,510 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:16:03,510 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Configure stats pid to 2106718 +2025-01-06 20:16:03,510 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/settings +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug.log +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug-internal.log +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:16:03,512 INFO MainThread:2106718 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg'} +2025-01-06 20:16:03,512 INFO MainThread:2106718 [wandb_init.py:init():619] starting backend +2025-01-06 20:16:03,512 INFO MainThread:2106718 [wandb_init.py:init():623] setting up manager +2025-01-06 20:16:03,516 INFO MainThread:2106718 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:16:03,517 INFO MainThread:2106718 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:16:03,530 INFO MainThread:2106718 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:16:03,537 INFO MainThread:2106718 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:16:03,936 INFO MainThread:2106718 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:16:04,006 INFO MainThread:2106718 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:16:04,007 INFO MainThread:2106718 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:16:04,354 INFO MainThread:2106718 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:16:04,355 INFO MainThread:2106718 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/tb, True +2025-01-06 20:16:08,208 INFO MainThread:2106718 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:20:55,764 WARNING MsgRouterThr:2106718 [router.py:message_loop():77] message_loop has been closed diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH/eval_robot.py b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebe4331c98fb22916121f7756e79b15b3abb02dd --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1013_1' +wandb_notes: + desc: null + value: 1013_1_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736212563 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/diff.patch b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..eaab2121dca194bbf5d21d04e789d724aa2ffaa3 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..36d45ad 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json new file mode 100644 index 0000000000000000000000000000000000000000..db88f8dce313fe228bb6c4839d50146f71aa6144 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 20, 1.0, 0.9946215747797684, 1, 0.8453640490079612, 1, 0.865594230519132, 1, 1, 1]]} \ No newline at end of file diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..099b9a8e43ab564deccf3e642bb5ec5173b03257 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77806420d75eb9ff49fc3a34e3fedb731f74f8ef2f5e25ca46e4ae0df0fdd3ce +size 1432277 diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a40be3bbb2d71134717ab95441ce41306fb5adb --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6260f5c906394d6a4f4979b7c8f81ff6cc2d39809dd6a858d9690bc77a3f53 +size 1389414 diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53c3f0c93073b654185b7bf3d61351d124cc95ba --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112b95f61f166bf491344b68561ce66a186890d9953c444be552658a09e883b0 +size 1427754 diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb3d9d424d856a362e300268da167422cb49706c --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0f62edae868114b9186f36bb50e06fbc82b1a68bedad4e05d70995d7d077fd2 +size 1427838 diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..339d3504322ba05f6b43d457d39ffcef2a685b0b --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e7800e9e899160f2ce3d7e9f51ec0a6b468ce999404b7fe00713cb6d021a83 +size 1470161 diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..84a47a2f7b0a0504ec646447357b69d144d5e463 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log @@ -0,0 +1,268 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736212568.209625]: resetting environment +[INFO] [1736212568.214443]: cleared current plan +[INFO] [1736212568.215383]: moving to home +[INFO] [1736212569.217270]: reached home +[INFO] [1736212569.218536]: reset action history +[INFO] [1736212571.928984]: environment reset +Starting episode 0 +[INFO] [1736212571.933493]: resetting environment +[INFO] [1736212571.937386]: cleared current plan +[INFO] [1736212571.938247]: moving to home +[INFO] [1736212572.940247]: reached home +[INFO] [1736212572.941547]: reset action history +[INFO] [1736212575.651425]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736212580.832834]: resetting environment +[INFO] [1736212580.837167]: cleared current plan +[INFO] [1736212580.838058]: moving to home +[INFO] [1736212581.839566]: reached home +[INFO] [1736212581.841247]: reset action history +[INFO] [1736212584.550198]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/episode_rosbags/episode_0_2025-01-06-20-16-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2106718.log +started roslaunch server http://158.130.50.37:35631/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2107009] +process[print_text-2]: started with pid [2107033] +process[rosbag_record-3]: started with pid [2107034] +started bagging! +[INFO] [1736212619.400698]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736212620.198041]: Stored episode 1. +[INFO] [1736212620.198327]: Saving video... +[INFO] [1736212620.404790]: Video saved! +Starting episode 1 +[INFO] [1736212620.416800]: resetting environment +[INFO] [1736212620.428606]: cleared current plan +[INFO] [1736212620.429010]: moving to home +[INFO] [1736212624.130808]: reached home +[INFO] [1736212624.131239]: reset action history +[INFO] [1736212626.893743]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212628.662800]: resetting environment +[INFO] [1736212628.668525]: cleared current plan +[INFO] [1736212628.669006]: moving to home +[INFO] [1736212629.670449]: reached home +[INFO] [1736212629.670905]: reset action history +[INFO] [1736212632.379678]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/episode_rosbags/episode_1_2025-01-06-20-17-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2106718.log +started roslaunch server http://158.130.50.37:37923/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2107160] +process[print_text-5]: started with pid [2107161] +process[rosbag_record-6]: started with pid [2107162] +started bagging! +For topic EE_pose: timestamp difference is 205774369 for nearest: 1736212635396435084 - target: 1736212635190660715 at idx 0 +For topic EE_pose: timestamp difference is 396603603 for nearest: 1736212635587264318 - target: 1736212635190660715 at idx 0 +[INFO] [1736212666.274024]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736212667.014987]: Stored episode 2. +[INFO] [1736212667.015197]: Saving video... +[INFO] [1736212667.646091]: Video saved! +Starting episode 2 +[INFO] [1736212667.672868]: resetting environment +[INFO] [1736212667.691586]: cleared current plan +[INFO] [1736212667.691815]: moving to home +[WARN] [1736212667.621765]: Plan exhausted +[WARN] [1736212667.659039]: Plan exhausted +[INFO] [1736212671.495932]: reached home +[INFO] [1736212671.498781]: reset action history +[INFO] [1736212674.224042]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212675.978575]: resetting environment +[INFO] [1736212675.986905]: cleared current plan +[INFO] [1736212675.987314]: moving to home +[INFO] [1736212676.988876]: reached home +[INFO] [1736212676.990360]: reset action history +[INFO] [1736212679.702328]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/episode_rosbags/episode_2_2025-01-06-20-17-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2106718.log +started roslaunch server http://158.130.50.37:36875/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2107326] +process[print_text-8]: started with pid [2107327] +process[rosbag_record-9]: started with pid [2107328] +started bagging! +[INFO] [1736212714.694114]: Storing episode... +[pub_text-7] killing on exit[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1736212715.460338]: Stored episode 3. +[INFO] [1736212715.462402]: Saving video... +[INFO] [1736212715.979703]: Video saved! +Starting episode 3 +[INFO] [1736212716.013543]: resetting environment +[INFO] [1736212716.024381]: cleared current plan +[INFO] [1736212716.026525]: moving to home +[INFO] [1736212719.731112]: reached home +[INFO] [1736212719.738034]: reset action history +[INFO] [1736212722.530112]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212731.862029]: resetting environment +[INFO] [1736212731.886284]: cleared current plan +[INFO] [1736212731.888883]: moving to home +[INFO] [1736212732.892648]: reached home +[INFO] [1736212732.895205]: reset action history +[INFO] [1736212735.669901]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/episode_rosbags/episode_3_2025-01-06-20-18-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2106718.log +started roslaunch server http://158.130.50.37:44311/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2107451] +process[print_text-11]: started with pid [2107452] +process[rosbag_record-12]: started with pid [2107476] +started bagging! +[INFO] [1736212770.562030]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736212771.362734]: Stored episode 4. +[INFO] [1736212771.370180]: Saving video... +[INFO] [1736212771.817389]: Video saved! +Starting episode 4 +[INFO] [1736212771.871265]: resetting environment +[INFO] [1736212771.889159]: cleared current plan +[INFO] [1736212771.889421]: moving to home +[INFO] [1736212775.693820]: reached home +[INFO] [1736212775.695716]: reset action history +[INFO] [1736212778.433665]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212781.390475]: resetting environment +[INFO] [1736212781.421347]: cleared current plan +[INFO] [1736212781.425383]: moving to home +[INFO] [1736212782.429208]: reached home +[INFO] [1736212782.430660]: reset action history +[INFO] [1736212785.217551]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/episode_rosbags/episode_4_2025-01-06-20-19-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2106718.log +started roslaunch server http://158.130.50.37:41751/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2107615] +process[print_text-14]: started with pid [2107616] +process[rosbag_record-15]: started with pid [2107617] +started bagging! +[INFO] [1736212821.265656]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736212822.017490]: Stored episode 5. +[INFO] [1736212822.022336]: Saving video... +[WARN] [1736212822.635928]: Plan exhausted +[WARN] [1736212822.675318]: Plan exhausted +[WARN] [1736212822.716932]: Plan exhausted +[WARN] [1736212822.755224]: Plan exhausted +[INFO] [1736212822.706829]: Video saved! +[INFO] [1736212822.744927]: resetting environment +[INFO] [1736212822.767574]: cleared current plan +[INFO] [1736212822.768039]: moving to home +[INFO] [1736212826.575782]: reached home +[INFO] [1736212826.586589]: reset action history +[INFO] [1736212829.385061]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-metadata.json b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..43400705c5d201a7cdded80800be024985c40b68 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:16:04.030562", + "startedAt": "2025-01-07T01:16:03.503587", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3747.769375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4484.452, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.003, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 462.5564193725586 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ec4deb36890a5b2d4406503573af6e2dc738cc58 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "77806420d75eb9ff49fc3a34e3fedb731f74f8ef2f5e25ca46e4ae0df0fdd3ce", "size": 1432277, "path": "media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4"}, "global_step": 665, "_timestamp": 1736212832.8167527, "_runtime": 269.2989685535431, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5f6260f5c906394d6a4f4979b7c8f81ff6cc2d39809dd6a858d9690bc77a3f53", "size": 1389414, "path": "media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "112b95f61f166bf491344b68561ce66a186890d9953c444be552658a09e883b0", "size": 1427754, "path": "media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f0f62edae868114b9186f36bb50e06fbc82b1a68bedad4e05d70995d7d077fd2", "size": 1427838, "path": "media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "57e7800e9e899160f2ce3d7e9f51ec0a6b468ce999404b7fe00713cb6d021a83", "size": 1470161, "path": "media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://dohquqfcd717n3xgxa9b3cihrlypjxa9dvgt6n0nlqln1oj7kgl8o8vi1h20pbi0tcyznfm8m3owtcfl9f2rg5ioco1f3mr7n6hwbho9uuegh8kcingd9my6co6kr6rr/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://91qrquuopwnmjvontq0klnd9jvqt14pwoehg6rpv2mfj7e2g52nt6rs5urxaxt4g5yqt40ytvknnw9b9xy9lcwyf32r1zwrdx0lvyx07hi9fptr7rjict6suegra3bux:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 20, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9946215747797684, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8453640490079612, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.865594230519132, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c68d11fd8ad6b790b7a949bb003b7716c3f8e083e007b71762bd414593b8472d", "size": 419, "artifact_path": "wandb-client-artifact://bn2odl3z5ljdeokk73os6y75va06lmera3961574wdacuuszqwvytzodsrbul2ajp4aqrfxwi8b6gs7fgv47kedm379snxs8me7mq15x5w8khnktmdvdxn8h3uo0b3q7/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://8jsxgppqrp7z234v64v7x104sju87hm8xp56sd83metxv2ra4fm7mrcmlh62j1gtlxfujsptvl811yjj9m71ndxmu1dz89ijvx8182fhbvme7i9rh7fyubj9xrzyqp4s:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 280}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9946215748786926, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8453640341758728, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.865594208240509, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug-internal.log b/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5ca4ee1ae4ef62bb4fdc16883bb2b197d457d03b --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug-internal.log @@ -0,0 +1,780 @@ +2025-01-06 20:16:03,517 INFO StreamThr :2106904 [internal.py:wandb_internal():85] W&B internal server running at pid: 2106904, started at: 2025-01-06 20:16:03.517246 +2025-01-06 20:16:03,518 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:16:03,531 INFO WriterThread:2106904 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/run-oagunwyg.wandb +2025-01-06 20:16:03,531 DEBUG SenderThread:2106904 [sender.py:send():379] send: header +2025-01-06 20:16:03,541 DEBUG SenderThread:2106904 [sender.py:send():379] send: run +2025-01-06 20:16:03,921 INFO SenderThread:2106904 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files +2025-01-06 20:16:03,921 INFO SenderThread:2106904 [sender.py:_start_run_threads():1188] run started: oagunwyg with start time 1736212563.517784 +2025-01-06 20:16:03,936 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:16:03,937 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:16:04,012 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:16:04,017 DEBUG HandlerThread:2106904 [system_info.py:__init__():26] System info init +2025-01-06 20:16:04,017 DEBUG HandlerThread:2106904 [system_info.py:__init__():41] System info init done +2025-01-06 20:16:04,017 INFO HandlerThread:2106904 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:16:04,017 INFO SystemMonitor:2106904 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:16:04,017 INFO HandlerThread:2106904 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:16:04,017 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:16:04,018 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:16:04,019 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:16:04,019 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:16:04,020 INFO SystemMonitor:2106904 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:16:04,030 DEBUG HandlerThread:2106904 [system_info.py:probe():152] Probing system +2025-01-06 20:16:04,031 DEBUG HandlerThread:2106904 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_info.py:probe():200] Probing system done +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:16:04.030562', 'startedAt': '2025-01-07T01:16:03.503587', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.769375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4484.452, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 462.5564193725586}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:16:04,036 INFO HandlerThread:2106904 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:16:04,036 INFO HandlerThread:2106904 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:16:04,036 DEBUG HandlerThread:2106904 [system_info.py:_save_code():44] Saving code +2025-01-06 20:16:04,038 DEBUG HandlerThread:2106904 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:16:04,038 DEBUG HandlerThread:2106904 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:16:04,099 DEBUG HandlerThread:2106904 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:16:04,100 INFO HandlerThread:2106904 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:16:04,103 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:16:04,104 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:16:04,104 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:16:04,105 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:16:04,352 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:16:04,352 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:16:04,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:04,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:04,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:04,355 INFO HandlerThread:2106904 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/tb" + save: true +} +control { + relay_id: "42b887af31504a9fb5f1f21b3fb07c55" +} +_info { + stream_id: "oagunwyg" +} + +2025-01-06 20:16:04,355 DEBUG HandlerThread:2106904 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:16:04,368 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/wbgygke7-code/FISH/eval_robot.py +2025-01-06 20:16:04,377 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/n8xz4rgy-wandb-metadata.json +2025-01-06 20:16:04,386 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/et0293sr-diff.patch +2025-01-06 20:16:04,509 DEBUG SenderThread:2106904 [sender.py:send():379] send: telemetry +2025-01-06 20:16:04,510 DEBUG SenderThread:2106904 [sender.py:send():379] send: tbrecord +2025-01-06 20:16:04,510 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:16:04,510 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 with policy live +2025-01-06 20:16:04,697 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/9i13xyj8-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/diff.patch +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-metadata.json +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH/eval_robot.py +2025-01-06 20:16:04,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code +2025-01-06 20:16:04,924 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH +2025-01-06 20:16:05,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:06,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:06,922 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:07,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:08,209 DEBUG SenderThread:2106904 [sender.py:send():379] send: config +2025-01-06 20:16:08,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:09,217 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:09,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:10,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:10,923 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:11,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:12,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:12,924 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:13,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:14,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:14,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:14,924 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:15,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:16,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:16,925 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:17,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:18,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:19,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:19,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:19,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:19,443 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:20,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:21,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:22,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:22,927 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:23,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:24,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:24,550 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:25,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:26,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:26,928 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:16:27,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:28,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:29,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:30,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:30,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:31,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:32,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:33,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:34,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:34,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:34,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:35,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:35,424 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:35,930 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml +2025-01-06 20:16:36,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:37,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:38,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:39,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:40,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:41,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:41,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:42,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:43,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:44,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:45,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:46,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:46,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:47,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:48,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:49,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:16:49,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:16:49,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:50,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:51,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:51,410 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:52,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:53,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:54,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:55,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:56,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:57,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:16:57,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:58,361 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:16:59,361 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:00,363 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:00,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:17:00,450 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:17:00,450 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 with policy now +2025-01-06 20:17:00,450 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:17:00,450 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:17:00,451 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:17:00,690 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/kov16fo0-media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos +2025-01-06 20:17:00,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media +2025-01-06 20:17:01,204 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/lqok3r61-media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:17:01,363 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:02,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:02,452 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:02,937 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:03,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:04,021 DEBUG SystemMonitor:2106904 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:17:04,021 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:17:04,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:04,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:04,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:05,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:06,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:06,939 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:07,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:07,937 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:08,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:08,939 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:08,940 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:17:09,180 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/adrkwjol-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:17:09,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:10,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:10,940 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:11,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:12,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:13,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:13,624 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:14,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:14,942 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:15,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:16,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:16,942 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:17,366 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:18,366 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:18,968 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:19,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:19,353 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:19,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:20,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:21,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:22,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:23,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:24,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:24,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:25,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:26,368 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:27,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:28,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:29,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:29,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:30,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:31,369 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:32,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:33,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:34,022 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:17:34,353 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:34,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:34,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:34,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:35,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:36,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:37,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:38,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:39,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:40,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:40,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:41,370 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:42,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:43,372 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:44,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:45,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:45,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:46,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:47,371 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:47,669 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:17:47,669 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 with policy now +2025-01-06 20:17:47,672 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:17:47,672 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:17:47,672 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:17:47,693 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:17:47,903 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/5tunx9nf-media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:17:47,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:17:47,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:17:47,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:17:48,222 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/gkxl7vwk-media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:17:48,373 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:48,953 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:49,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:17:49,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:17:49,375 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:49,954 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:50,374 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:50,468 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:51,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:52,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:52,955 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:53,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:54,376 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:55,376 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:55,967 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:17:56,376 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:56,774 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:17:56,774 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:17:56,774 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:17:56,956 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:17:56,956 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:56,956 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:17:57,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:58,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:17:58,957 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:17:59,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:00,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:00,957 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:01,106 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:01,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:02,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:03,377 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:04,022 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:18:04,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:04,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:04,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:05,378 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:06,378 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:06,455 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:07,378 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:08,379 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:09,379 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:10,379 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:11,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:12,357 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:12,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:13,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:14,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:15,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:16,380 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:17,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:17,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:18,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:19,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:19,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:19,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:20,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:21,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:22,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:22,452 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:23,381 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:24,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:25,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:26,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:27,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:28,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:28,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:29,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:30,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:31,382 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:32,383 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:33,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:33,383 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:34,023 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:18:34,354 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:34,354 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:34,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:35,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:36,011 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:18:36,011 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 with policy now +2025-01-06 20:18:36,011 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:18:36,012 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:18:36,012 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:18:36,025 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:18:36,257 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/5un1at3d-media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:18:36,386 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:36,986 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:18:36,987 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:36,987 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:18:36,987 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:18:37,280 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/papyxprm-media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:18:37,391 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:38,396 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:39,027 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:39,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:40,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:40,988 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:41,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:42,395 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:43,393 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:44,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:44,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:44,989 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:45,395 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:46,398 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:47,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:48,395 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:49,358 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:18:49,358 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:18:49,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:49,503 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:50,399 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:51,396 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:51,992 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:18:52,245 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/rtcnlw8t-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:18:52,397 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:52,619 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:18:52,619 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:18:52,619 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:18:52,992 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:18:52,993 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:53,400 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:54,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:54,993 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:55,362 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:18:55,405 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:56,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:56,993 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:18:57,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:58,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:18:59,401 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:00,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:01,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:01,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:02,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:03,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:04,023 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:19:04,355 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:04,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:04,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:05,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:06,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:06,493 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:07,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:08,402 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:09,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:10,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:11,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:12,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:12,403 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:13,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:14,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:15,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:16,404 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:17,405 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:18,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:18,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:19,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:19,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:19,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:20,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:21,406 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:22,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:23,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:23,412 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:24,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:25,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:26,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:27,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:28,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:29,360 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:29,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:30,407 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:31,417 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:31,865 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:19:31,865 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 with policy now +2025-01-06 20:19:31,866 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:19:31,868 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:19:31,868 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:19:31,879 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:19:32,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:19:32,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:19:32,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:19:32,098 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/zx2xu9fj-media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:19:32,361 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/ki6766a9-media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:19:32,412 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:33,006 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:33,418 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:34,024 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:19:34,359 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:34,359 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:34,416 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:34,434 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:35,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:36,423 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:37,007 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:37,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:38,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:39,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:40,367 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:40,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:41,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:41,425 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:41,675 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:19:41,675 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:19:41,676 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:19:42,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:19:42,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:19:42,425 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:43,009 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:43,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:44,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:45,010 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:45,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:46,261 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:46,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:47,011 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:47,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:48,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:49,012 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:19:49,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:19:49,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:19:49,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:50,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:51,426 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:51,466 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:52,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:53,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:54,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:55,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:56,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:57,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:19:57,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:58,427 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:19:59,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:00,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:01,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:02,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:02,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:03,428 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:04,024 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:20:04,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:20:04,356 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:20:04,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:05,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:06,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:07,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:07,481 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:08,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:09,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:10,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:11,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:12,429 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:13,364 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:13,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:14,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:15,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:16,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:17,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:18,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:18,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:19,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:20:19,357 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:20:19,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:20,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:21,430 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:22,447 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:22,740 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:20:22,740 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 with policy now +2025-01-06 20:20:22,744 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:20:22,745 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:22,745 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:22,754 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:22,978 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/qhmdiuow-media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:23,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:23,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:23,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:23,025 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval +2025-01-06 20:20:23,303 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/2n4ohrra-media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:23,439 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:23,770 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:24,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:24,452 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:25,024 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:25,445 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:26,447 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:27,445 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:28,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:29,026 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:29,374 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:29,450 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:30,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:31,026 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:31,455 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:32,027 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:32,273 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/rnz4c44e-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:32,325 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:20:32,326 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:20:32,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:32,522 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:20:32,805 INFO wandb-upload_1:2106904 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpiq9wfd72 +2025-01-06 20:20:32,897 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:20:32,907 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:20:33,027 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:33,027 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval +2025-01-06 20:20:33,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table +2025-01-06 20:20:33,171 INFO SenderThread:2106904 [sender.py:send_request_log_artifact():1525] logged artifact run-oagunwyg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI0ODQ2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYxNDEzNA==', 'latestArtifact': None}} +2025-01-06 20:20:33,171 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:20:33,171 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 20:20:33,171 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:33,172 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,172 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,172 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:33,172 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,173 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,173 DEBUG SenderThread:2106904 [sender.py:send():379] send: telemetry +2025-01-06 20:20:33,173 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,173 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,173 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,174 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,174 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,174 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,174 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,174 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,175 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,175 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,175 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,175 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,176 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,176 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,176 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,176 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,177 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,177 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,177 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,177 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,178 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,178 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:33,178 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:20:33,351 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/ga6vwjro-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:33,373 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/9wbv3zfg-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:33,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:33,590 INFO wandb-upload_1:2106904 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1833up9y +2025-01-06 20:20:33,595 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/wwlds7kz-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:33,973 INFO SenderThread:2106904 [sender.py:send_request_log_artifact():1525] logged artifact run-oagunwyg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI0ODU1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYxNDIyMA==', 'latestArtifact': None}} +2025-01-06 20:20:33,973 DEBUG SenderThread:2106904 [sender.py:send():379] send: files +2025-01-06 20:20:33,973 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-06 20:20:33,973 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:33,973 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:33,974 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:34,026 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:20:34,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:34,217 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/i35iyhl9-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:34,356 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:20:34,357 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:20:34,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:34,469 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:35,028 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:35,448 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:36,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:37,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:38,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:39,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:40,365 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:40,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:41,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:42,449 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:43,450 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:44,450 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:44,534 DEBUG SenderThread:2106904 [sender.py:send():379] send: exit +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:send_exit():588] handling runtime: 280 +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:44,534 INFO SenderThread:2106904 [sender.py:send_exit():594] send defer +2025-01-06 20:20:44,534 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,534 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:20:44,535 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:20:44,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,535 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:20:44,535 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:20:44,535 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:20:44,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,535 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:20:44,535 INFO HandlerThread:2106904 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:20:44,536 DEBUG SystemMonitor:2106904 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:20:44,536 DEBUG SystemMonitor:2106904 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:20:44,536 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:20:44,537 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:20:44,589 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:20:44,590 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:20:44,590 INFO HandlerThread:2106904 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:20:44,590 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,590 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:20:44,590 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:20:44,590 DEBUG SenderThread:2106904 [sender.py:send():379] send: stats +2025-01-06 20:20:44,591 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,591 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:20:44,591 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:44,591 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:20:44,591 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:20:44,591 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:44,591 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:20:45,032 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:47,033 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:20:50,745 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:50,746 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,746 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:20:50,746 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:20:50,746 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:50,746 DEBUG SenderThread:2106904 [sender.py:send():379] send: history +2025-01-06 20:20:50,747 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:20:50,747 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:50,747 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:20:50,748 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:50,748 DEBUG SenderThread:2106904 [sender.py:send():379] send: summary +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:20:50,752 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:20:50,752 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:50,752 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:20:50,752 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,752 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:20:50,756 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:20:50,878 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:20:50,878 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:50,878 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:20:50,878 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:50,878 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:20:51,035 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:51,035 INFO Thread-12 :2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml +2025-01-06 20:20:51,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:53,793 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:20:53,793 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:53,793 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:53,794 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:20:53,794 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:53,794 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:20:53,794 INFO SenderThread:2106904 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:20:53,796 INFO SenderThread:2106904 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:20:53,798 INFO SenderThread:2106904 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:20:53,808 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:20:53,808 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:53,808 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:20:53,809 DEBUG SenderThread:2106904 [sender.py:send():379] send: artifact +2025-01-06 20:20:54,026 INFO SenderThread:2106904 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk3NDE3OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:20:54,026 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,027 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:20:54,027 INFO SenderThread:2106904 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml config.yaml +2025-01-06 20:20:54,036 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json wandb-summary.json +2025-01-06 20:20:54,037 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt requirements.txt +2025-01-06 20:20:54,039 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:20:54,040 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:54,040 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log output.log +2025-01-06 20:20:54,040 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/diff.patch diff.patch +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 media/videos/eval/3_eval_3_f0f62edae868114b9186.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 media/videos/eval/4_eval_4_57e7800e9e899160f2ce.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 media/videos/eval/1_eval_1_5f6260f5c906394d6a4f.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 media/videos/eval/0_eval_0_77806420d75eb9ff49fc.mp4 +2025-01-06 20:20:54,042 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 media/videos/eval/2_eval_2_112b95f61f166bf49134.mp4 +2025-01-06 20:20:54,043 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-06 20:20:54,043 INFO SenderThread:2106904 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 20:20:54,043 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:20:54,044 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,044 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:20:54,046 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,046 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:20:54,046 INFO SenderThread:2106904 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:20:54,225 INFO wandb-upload_2:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/requirements.txt +2025-01-06 20:20:54,232 INFO wandb-upload_1:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/wandb-summary.json +2025-01-06 20:20:54,252 INFO wandb-upload_0:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/config.yaml +2025-01-06 20:20:54,280 INFO wandb-upload_4:2106904 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/files/output.log +2025-01-06 20:20:54,309 INFO wandb-upload_3:2106904 [upload_job.py:push():130] Uploaded file /tmp/tmpftltjywtwandb/osqx1d02-events.out.tfevents.1736212564.leonmkim-ROG-Strix-G15CS-G15CS.2106718.0 +2025-01-06 20:20:54,509 INFO Thread-11 (_thread_body):2106904 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:20:54,509 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,509 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:20:54,510 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,510 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:20:54,510 INFO SenderThread:2106904 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:20:54,511 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:20:54,511 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,511 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:20:54,511 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,511 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:20:54,511 INFO SenderThread:2106904 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:20:54,535 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:20:54,714 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:54,714 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,714 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:20:54,714 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:20:54,714 INFO SenderThread:2106904 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:20:54,715 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:20:54,715 INFO HandlerThread:2106904 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:20:54,715 DEBUG SenderThread:2106904 [sender.py:send():379] send: final +2025-01-06 20:20:54,715 DEBUG SenderThread:2106904 [sender.py:send():379] send: footer +2025-01-06 20:20:54,715 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: defer +2025-01-06 20:20:54,715 INFO SenderThread:2106904 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:20:54,715 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:54,716 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:20:54,716 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:20:54,716 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:54,716 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:20:54,716 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:20:54,717 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:20:54,719 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:20:54,763 DEBUG SenderThread:2106904 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:20:54,763 INFO MainThread:2106904 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:20:54,763 INFO MainThread:2106904 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:20:54,764 INFO MainThread:2106904 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:20:54,764 DEBUG HandlerThread:2106904 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:20:54,764 INFO HandlerThread:2106904 [handler.py:finish():882] shutting down handler +2025-01-06 20:20:55,716 INFO WriterThread:2106904 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/run-oagunwyg.wandb +2025-01-06 20:20:55,763 INFO SenderThread:2106904 [sender.py:finish():1615] shutting down sender +2025-01-06 20:20:55,763 INFO SenderThread:2106904 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:20:55,763 INFO SenderThread:2106904 [file_pusher.py:join():175] waiting for file pusher diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug.log b/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4979ddd98971c54f7f3e93f4e07185274de46067 --- /dev/null +++ b/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:16:03,510 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:16:03,510 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Configure stats pid to 2106718 +2025-01-06 20:16:03,510 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/settings +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug.log +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/wandb/run-20250106_201603-oagunwyg/logs/debug-internal.log +2025-01-06 20:16:03,511 INFO MainThread:2106718 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:16:03,512 INFO MainThread:2106718 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg'} +2025-01-06 20:16:03,512 INFO MainThread:2106718 [wandb_init.py:init():619] starting backend +2025-01-06 20:16:03,512 INFO MainThread:2106718 [wandb_init.py:init():623] setting up manager +2025-01-06 20:16:03,516 INFO MainThread:2106718 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:16:03,517 INFO MainThread:2106718 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:16:03,530 INFO MainThread:2106718 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:16:03,537 INFO MainThread:2106718 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:16:03,936 INFO MainThread:2106718 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:16:04,006 INFO MainThread:2106718 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:16:04,007 INFO MainThread:2106718 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:16:04,353 INFO MainThread:2106718 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:16:04,354 INFO MainThread:2106718 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:16:04,355 INFO MainThread:2106718 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/oagunwyg/tb, True +2025-01-06 20:16:08,208 INFO MainThread:2106718 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:20:55,764 WARNING MsgRouterThr:2106718 [router.py:message_loop():77] message_loop has been closed diff --git a/oagunwyg/wandb/run-20250106_201603-oagunwyg/run-oagunwyg.wandb b/oagunwyg/wandb/run-20250106_201603-oagunwyg/run-oagunwyg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0f82205fe82105a4796676d49fa20d6a4c14ba99 Binary files /dev/null and b/oagunwyg/wandb/run-20250106_201603-oagunwyg/run-oagunwyg.wandb differ diff --git a/ul5e771y/episode_rosbags/cam_tf_world.npy b/ul5e771y/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/ul5e771y/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/ul5e771y/episode_rosbags/depth_K.npy b/ul5e771y/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/ul5e771y/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/ul5e771y/eval_robot.log b/ul5e771y/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ab70259e62d6a3921d61a39076cc3712bf808d67 --- /dev/null +++ b/ul5e771y/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 18:17:38,085][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 18:17:38,088][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/vhqg7j1c/episode_rosbags/episode_3_2025-01-06-17-58-15.bag b/vhqg7j1c/episode_rosbags/episode_3_2025-01-06-17-58-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..4bf8a8186278a9d931bb6742af1c6f91ebf82c3d --- /dev/null +++ b/vhqg7j1c/episode_rosbags/episode_3_2025-01-06-17-58-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd75068963c2e96e544b03e9f80ae835c6e51186d849591cc8f39db311e6b100 +size 2602412108 diff --git a/zk6ipw9e/episode_rosbags/episode_1_2025-01-07-16-35-28.bag b/zk6ipw9e/episode_rosbags/episode_1_2025-01-07-16-35-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..6afabb1328b3a9f9b06d497315efcbf28e7b0c51 --- /dev/null +++ b/zk6ipw9e/episode_rosbags/episode_1_2025-01-07-16-35-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e1964796779a0d84e44677d079ca18d0ce911d5f899f69f6d29e53d99e4f54 +size 2608297683 diff --git a/zk6ipw9e/episode_rosbags/episode_2_2025-01-07-16-36-18.bag b/zk6ipw9e/episode_rosbags/episode_2_2025-01-07-16-36-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..b531994c7c84ac442014247b37e16f8a9d266b6c --- /dev/null +++ b/zk6ipw9e/episode_rosbags/episode_2_2025-01-07-16-36-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78e723fc935442f2fd64fcffb35322944e248e965432047e84e941a58c73c83b +size 2599083005 diff --git a/zk6ipw9e/episode_rosbags/episode_3_2025-01-07-16-37-18.bag b/zk6ipw9e/episode_rosbags/episode_3_2025-01-07-16-37-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..3a9b82292e1823ebe4ef6adcc3af7b66778970df --- /dev/null +++ b/zk6ipw9e/episode_rosbags/episode_3_2025-01-07-16-37-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59707c1535f14cdb0f3d8049e2da67f4badcd52daada2d31bc1044f87b9aaffc +size 2607787254 diff --git a/zk6ipw9e/episode_rosbags/episode_4_2025-01-07-16-38-21.bag b/zk6ipw9e/episode_rosbags/episode_4_2025-01-07-16-38-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..9bf15e65968c9230efd87f0f4f4539a7f29c11f7 --- /dev/null +++ b/zk6ipw9e/episode_rosbags/episode_4_2025-01-07-16-38-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa8976b0a53d8a3a6bd2b67a6f60392c4484c1223c1e30b0dba3c358da2d7234 +size 2602363898