diff --git a/.gitattributes b/.gitattributes index 5d716cc238f0e8cba315874cf7b6378e4fe7ca61..7458463688071aa094a960f19db8bf9758448b3a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -63,3 +63,31 @@ y2m69p6z/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text y2m69p6z/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text y2m69p6z/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text y2m69p6z/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 filter=lfs diff=lfs merge=lfs -text +uymqqs40/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +y2m69p6z/episode_rosbags/episode_4_2025-01-04-20-47-19.bag filter=lfs diff=lfs merge=lfs -text +l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l4rsu423/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +uymqqs40/episode_rosbags/episode_0_2025-01-04-22-26-57.bag filter=lfs diff=lfs merge=lfs -text +y2m69p6z/episode_rosbags/episode_2_2025-01-04-20-45-26.bag filter=lfs diff=lfs merge=lfs -text +y2m69p6z/episode_rosbags/episode_0_2025-01-04-20-43-31.bag filter=lfs diff=lfs merge=lfs -text +l4rsu423/episode_rosbags/episode_3_2025-01-03-22-55-32.bag filter=lfs diff=lfs merge=lfs -text +pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/c3bfn6bi/.hydra/config.yaml b/c3bfn6bi/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/c3bfn6bi/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/c3bfn6bi/.hydra/hydra.yaml b/c3bfn6bi/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d790533ee04a65238e671a6c10b643d1cda24cb2 --- /dev/null +++ b/c3bfn6bi/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/c3bfn6bi + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/c3bfn6bi/.hydra/overrides.yaml b/c3bfn6bi/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/c3bfn6bi/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/c3bfn6bi/eval_policy.log b/c3bfn6bi/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..72896687e587650f0e4c8c56bbbcc48480bc4e22 --- /dev/null +++ b/c3bfn6bi/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 20:37:23,874][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:37:23,877][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 20:37:27,374][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/l4rsu423/.hydra/config.yaml b/l4rsu423/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/l4rsu423/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/l4rsu423/.hydra/hydra.yaml b/l4rsu423/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3ec9350319d9a9e8d8501ad36e61aa499c5a5e0 --- /dev/null +++ b/l4rsu423/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/l4rsu423/.hydra/overrides.yaml b/l4rsu423/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/l4rsu423/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/l4rsu423/episode_rosbags/aligned_depth_to_color_K.npy b/l4rsu423/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/l4rsu423/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/l4rsu423/episode_rosbags/cam_tf_world.npy b/l4rsu423/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/l4rsu423/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/l4rsu423/episode_rosbags/color_K.npy b/l4rsu423/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/l4rsu423/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/l4rsu423/episode_rosbags/depth_K.npy b/l4rsu423/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/l4rsu423/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/l4rsu423/episode_rosbags/episode_3_2025-01-03-22-55-32.bag b/l4rsu423/episode_rosbags/episode_3_2025-01-03-22-55-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..0e65a63329c3b38eb6fe90bd8444953165b82eec --- /dev/null +++ b/l4rsu423/episode_rosbags/episode_3_2025-01-03-22-55-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:200d3473bf32e7f341ab9c78d0977feeb0591c76b2fb9e0e0050a2a5061253ff +size 1835064759 diff --git a/l4rsu423/eval_robot.log b/l4rsu423/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..451badc6f518e9d5a3c865309fcba154c5a08604 --- /dev/null +++ b/l4rsu423/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:52:24,247][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:52:24,250][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/l4rsu423/eval_video/0_eval.mp4 b/l4rsu423/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..adc783ee72d90cd1cf143375ab8177314e772cf6 --- /dev/null +++ b/l4rsu423/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ec425df4891952a54d5fb9d467857d25a196a5aca8b03784f4fb0f903ee1c9 +size 1443427 diff --git a/l4rsu423/eval_video/1_eval.mp4 b/l4rsu423/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1838dc4e3b9820f65682062b91dfaaf139b6270f --- /dev/null +++ b/l4rsu423/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4475b9280ad87e0fbb8278e792cc867599d0604b322f710717190f93ff733c1 +size 1576912 diff --git a/l4rsu423/eval_video/2_eval.mp4 b/l4rsu423/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac023ebd6f5d74ba35fff8aa971098bb186cb62d --- /dev/null +++ b/l4rsu423/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cf06e2e8c9f28608fdd7cc380d343def13a9bf81c3974a35365fd5df61c59b2 +size 1429048 diff --git a/l4rsu423/eval_video/3_eval.mp4 b/l4rsu423/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2080f3ea197fc28691477778cb4296084170fd1a --- /dev/null +++ b/l4rsu423/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5735b0e0eaefce2d8e6d528b21bf9dcec1dd65b8aed354e75d02a35abb01e014 +size 1120327 diff --git a/l4rsu423/eval_video/4_eval.mp4 b/l4rsu423/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..19ec1fa2fcd3e356426ba092d51546463d36d15b --- /dev/null +++ b/l4rsu423/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fd61d3f24849abbec1d0c7d2d1065ce044cc1e0f85a78d84588f2a99b2b4478 +size 1475550 diff --git a/l4rsu423/tb/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 b/l4rsu423/tb/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 new file mode 100644 index 0000000000000000000000000000000000000000..a9f3a9fb528e0aac422f56c968853ba6544ca617 --- /dev/null +++ b/l4rsu423/tb/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5574220b59dc8427b65326f3e97fe244b32d20f9e59fb89b7617ae1350f7debb +size 6539 diff --git a/l4rsu423/wandb/debug-internal.log b/l4rsu423/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7f3a2f6581bef9cca7aa6dfe0f4b28eabdcfc9f8 --- /dev/null +++ b/l4rsu423/wandb/debug-internal.log @@ -0,0 +1,853 @@ +2025-01-03 22:52:29,994 INFO StreamThr :1989491 [internal.py:wandb_internal():85] W&B internal server running at pid: 1989491, started at: 2025-01-03 22:52:29.993802 +2025-01-03 22:52:29,995 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:52:30,007 INFO WriterThread:1989491 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/run-l4rsu423.wandb +2025-01-03 22:52:30,007 DEBUG SenderThread:1989491 [sender.py:send():379] send: header +2025-01-03 22:52:30,016 DEBUG SenderThread:1989491 [sender.py:send():379] send: run +2025-01-03 22:52:30,256 INFO SenderThread:1989491 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files +2025-01-03 22:52:30,257 INFO SenderThread:1989491 [sender.py:_start_run_threads():1188] run started: l4rsu423 with start time 1735962749.994182 +2025-01-03 22:52:30,271 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:52:30,271 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:52:30,319 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:52:30,324 DEBUG HandlerThread:1989491 [system_info.py:__init__():26] System info init +2025-01-03 22:52:30,324 DEBUG HandlerThread:1989491 [system_info.py:__init__():41] System info init done +2025-01-03 22:52:30,324 INFO HandlerThread:1989491 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:52:30,324 INFO SystemMonitor:1989491 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:52:30,324 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:52:30,325 INFO HandlerThread:1989491 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:52:30,325 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:52:30,325 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:52:30,326 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:52:30,327 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:52:30,338 DEBUG HandlerThread:1989491 [system_info.py:probe():152] Probing system +2025-01-03 22:52:30,341 DEBUG HandlerThread:1989491 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_info.py:probe():200] Probing system done +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:52:30.338917', 'startedAt': '2025-01-04T03:52:29.985396', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.9480000000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.24, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.364, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 534.1734809875488}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:52:30,347 INFO HandlerThread:1989491 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:52:30,347 INFO HandlerThread:1989491 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_info.py:_save_code():44] Saving code +2025-01-03 22:52:30,348 DEBUG HandlerThread:1989491 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:52:30,348 DEBUG HandlerThread:1989491 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:52:30,410 DEBUG HandlerThread:1989491 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:52:30,410 INFO HandlerThread:1989491 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:52:30,413 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:52:30,413 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:52:30,413 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:52:30,414 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:52:30,652 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:52:30,652 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:52:30,654 DEBUG SenderThread:1989491 [sender.py:send():379] send: telemetry +2025-01-03 22:52:30,684 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/0urikxzw-diff.patch +2025-01-03 22:52:30,686 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/ihtde9je-code/FISH/eval_robot.py +2025-01-03 22:52:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:52:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:30,697 INFO HandlerThread:1989491 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/tb" + save: true +} +control { + relay_id: "ab0a36204cd14eae8020fc5f5ed010c6" +} +_info { + stream_id: "l4rsu423" +} + +2025-01-03 22:52:30,697 DEBUG HandlerThread:1989491 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:52:30,698 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:52:30,704 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/t7zobwhm-wandb-metadata.json +2025-01-03 22:52:30,752 DEBUG SenderThread:1989491 [sender.py:send():379] send: tbrecord +2025-01-03 22:52:30,752 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:52:30,752 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 with policy live +2025-01-03 22:52:30,939 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/2d936hl5-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:52:31,258 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/diff.patch +2025-01-03 22:52:31,258 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt +2025-01-03 22:52:31,258 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH/eval_robot.py +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-metadata.json +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code +2025-01-03 22:52:31,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:32,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:33,257 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:33,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:34,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:35,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:35,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:36,625 DEBUG SenderThread:1989491 [sender.py:send():379] send: config +2025-01-03 22:52:36,669 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:37,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:38,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:39,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:39,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:40,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:40,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:41,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:42,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:43,260 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:43,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:44,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:45,261 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:45,652 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:52:45,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:52:45,693 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:45,729 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:46,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:47,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:48,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:49,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:50,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:50,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:51,263 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:51,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:52,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:53,263 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:53,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:54,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:55,264 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:55,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:56,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:56,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:57,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:58,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:59,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:00,652 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:00,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:00,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:01,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:01,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:02,266 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml +2025-01-03 22:53:02,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:03,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:04,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:05,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:06,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:07,132 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:07,267 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:07,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:08,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:09,268 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:09,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:10,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:11,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:12,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:12,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:13,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:14,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:15,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:15,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:15,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:16,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:17,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:17,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:18,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:19,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:20,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:21,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:22,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:23,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:23,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:24,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:25,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:26,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:27,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:28,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:28,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:29,577 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:53:29,577 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 with policy now +2025-01-03 22:53:29,578 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:53:29,578 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:53:29,579 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:53:29,580 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:53:29,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:29,838 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/3w8ct9ld-media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:53:30,273 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:53:30,327 DEBUG SystemMonitor:1989491 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:53:30,328 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:53:30,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:30,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:31,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:31,304 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/u2mbzarq-media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:53:31,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:32,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:33,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:34,237 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:34,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:35,275 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:35,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:36,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:37,276 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:37,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:38,276 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:53:38,513 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/wxznogyx-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:53:38,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:39,277 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:39,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:40,121 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:40,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:41,277 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:41,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:42,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:43,278 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:43,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:44,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:45,279 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:45,357 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:45,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:45,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:45,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:46,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:47,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:48,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:49,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:50,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:50,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:51,280 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:51,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:52,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:53,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:54,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:55,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:55,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:56,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:57,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:58,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:59,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:00,331 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:54:00,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:00,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:00,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:00,734 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:01,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:02,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:03,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:04,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:05,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:06,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:06,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:07,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:08,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:09,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:10,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:11,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:11,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:12,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:13,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:14,669 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:15,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:15,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:15,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:16,669 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:17,326 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:17,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:17,714 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:54:17,728 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 with policy now +2025-01-03 22:54:17,757 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:54:17,765 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:54:17,765 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:54:17,772 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:54:17,990 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/u77yxz4q-media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:54:18,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:54:18,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:54:18,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:54:18,548 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/n9afql6n-media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:54:18,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:19,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:19,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:20,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:21,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:22,421 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:22,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:23,291 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:23,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:24,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:25,291 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:25,670 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:26,670 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:27,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:27,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:28,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:29,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:30,329 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:54:30,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:30,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:31,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:32,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:32,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:33,672 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:34,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:35,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:36,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:37,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:38,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:38,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:39,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:40,125 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:54:40,125 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:54:40,126 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:54:40,296 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:54:40,296 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:54:40,529 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/6qenl5r4-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:54:40,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:41,296 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:41,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:42,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:43,297 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:43,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:43,742 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:44,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:45,298 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:45,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:45,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:45,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:46,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:47,299 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:47,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:48,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:48,764 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:49,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:50,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:51,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:52,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:53,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:54,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:54,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:55,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:56,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:57,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:58,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:59,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:59,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:00,329 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:55:00,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:00,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:00,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:01,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:02,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:03,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:04,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:04,783 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:05,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:06,680 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:07,680 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:08,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:09,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:10,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:10,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:11,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:12,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:13,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:14,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:15,308 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:15,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:15,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:15,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:15,715 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:16,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:17,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:18,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:19,309 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:19,500 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:55:19,501 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 with policy now +2025-01-03 22:55:19,501 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:55:19,502 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:55:19,502 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:55:19,512 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:55:19,689 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:19,744 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/hza77wkz-media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:55:20,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:55:20,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:55:20,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:55:20,568 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/pzwu3bx7-media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:55:20,687 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:21,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:21,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:21,699 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:22,694 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:23,702 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:24,696 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:25,312 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:25,695 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:26,695 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:27,109 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:27,312 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:27,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:28,313 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:55:28,699 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:28,964 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:55:28,964 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:55:28,964 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:55:29,313 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:29,313 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:55:29,701 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:30,330 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:55:30,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:30,657 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:30,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:31,314 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:31,720 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:32,208 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:32,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:33,314 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:33,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:34,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:35,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:36,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:37,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:37,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:38,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:39,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:40,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:41,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:42,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:43,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:43,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:44,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:45,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:45,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:45,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:46,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:47,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:48,705 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:49,320 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:49,357 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:49,705 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:50,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:51,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:52,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:53,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:54,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:54,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:55,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:56,717 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:57,729 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:57,892 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:55:57,892 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 with policy now +2025-01-03 22:55:57,895 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:55:57,895 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:55:57,895 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:55:57,896 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:55:58,143 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/fki19acd-media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:55:58,323 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:55:58,324 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:55:58,324 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:55:58,573 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/a36ouylc-media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:55:58,736 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:59,324 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:59,730 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:59,933 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:00,330 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:56:00,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:00,661 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:00,728 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:01,732 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:02,733 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:03,325 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:03,739 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:04,738 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:04,977 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:05,326 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:05,738 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:06,740 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:07,749 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:08,751 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:09,751 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:10,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:10,750 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:11,756 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:12,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:13,750 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:14,751 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:15,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:15,655 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:15,754 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:15,761 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:16,756 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:17,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:18,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:19,757 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:20,758 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:21,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:21,759 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:22,760 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:23,759 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:24,756 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:25,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:26,678 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:26,760 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:27,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:28,761 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:29,764 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:30,330 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:56:30,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:30,671 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:30,766 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:31,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:31,769 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:32,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:33,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:34,766 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:35,772 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:36,771 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:37,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:37,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:38,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:39,781 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:40,776 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:41,780 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:42,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:43,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:43,779 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:44,777 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:45,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:45,660 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:45,779 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:46,777 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:47,779 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:48,748 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:48,777 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:49,821 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:50,061 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:56:50,061 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:56:50,062 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:56:50,341 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:56:50,341 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:56:50,579 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/ak3oaw6t-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:56:50,791 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:51,341 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:51,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:52,791 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:53,342 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:53,782 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:53,926 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:54,782 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:55,342 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:55,782 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:56,783 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:57,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:58,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:59,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:59,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:00,331 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:57:00,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:00,658 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:00,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:01,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:02,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:03,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:04,734 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:04,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:05,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:06,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:07,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:08,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:09,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:10,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:10,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:11,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:12,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:13,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:14,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:15,350 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:15,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:15,658 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:15,729 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:15,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:16,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:17,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:18,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:19,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:20,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:21,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:21,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:22,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:23,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:24,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:25,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:26,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:27,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:27,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:28,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:29,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:29,869 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:57:29,869 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:57:29,870 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 with policy now +2025-01-03 22:57:29,870 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:29,870 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:29,871 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:30,135 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/5xw1l5rs-media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:57:30,332 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:57:30,355 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:30,356 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:57:30,356 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:57:30,628 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/1w30f7g5-media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:57:30,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:30,675 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:30,796 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:31,356 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:31,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:32,797 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:32,832 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:33,798 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:34,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:35,357 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:35,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:36,809 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:37,358 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:37,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:38,377 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:38,815 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:39,811 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:39,916 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:57:39,916 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:57:40,049 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:57:40,359 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:57:40,359 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:57:40,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media +2025-01-03 22:57:40,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval +2025-01-03 22:57:40,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table +2025-01-03 22:57:40,375 INFO wandb-upload_2:1989491 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp49ty0kqa +2025-01-03 22:57:40,434 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:57:40,442 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:57:40,542 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/rtxoxcw8-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:57:40,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:40,833 INFO SenderThread:1989491 [sender.py:send_request_log_artifact():1525] logged artifact run-l4rsu423-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzMzMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MjUzNA==', 'latestArtifact': None}} +2025-01-03 22:57:40,833 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:57:40,833 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-03 22:57:40,833 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:40,833 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,834 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,834 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:40,834 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,838 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,838 DEBUG SenderThread:1989491 [sender.py:send():379] send: telemetry +2025-01-03 22:57:40,838 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,839 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,839 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,839 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,839 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,840 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,840 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,840 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,840 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,841 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,841 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,841 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,841 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,842 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,842 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,842 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,842 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,843 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,843 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,843 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,843 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,844 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,844 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:57:41,078 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/3ap8bgdi-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:57:41,359 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval +2025-01-03 22:57:41,379 INFO wandb-upload_0:1989491 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnqrwh20p +2025-01-03 22:57:41,581 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/nbblimeg-media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:57:41,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:41,815 INFO SenderThread:1989491 [sender.py:send_request_log_artifact():1525] logged artifact run-l4rsu423-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzM0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MjUzNQ==', 'latestArtifact': None}} +2025-01-03 22:57:41,815 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:57:41,815 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json with policy now +2025-01-03 22:57:41,815 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:41,815 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:41,816 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:42,085 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/buhdm5bf-media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:57:42,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:42,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:43,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:43,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:43,816 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:44,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:45,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:45,660 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:45,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:46,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:47,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:48,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:49,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:49,812 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:50,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:51,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:52,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:53,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:54,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:55,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:55,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:56,365 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml +2025-01-03 22:57:56,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:57,574 DEBUG SenderThread:1989491 [sender.py:send():379] send: exit +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:send_exit():588] handling runtime: 327 +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:send_exit():594] send defer +2025-01-03 22:57:57,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:57:57,575 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:57:57,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:57:57,575 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:57:57,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:57:57,576 DEBUG SystemMonitor:1989491 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:57:57,576 DEBUG SystemMonitor:1989491 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:57:57,576 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:57:57,576 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:57:57,629 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:57:57,630 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:57:57,630 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:57:57,630 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,630 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:57:57,630 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:57:57,630 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:57:57,631 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,631 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:57:57,631 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,631 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:57:57,631 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:57:57,631 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,631 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:57:58,366 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:59,367 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:58:04,128 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:04,128 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:04,128 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:58:04,130 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,130 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:58:04,130 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:58:04,130 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:04,130 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:58:04,130 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:04,131 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:58:04,131 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:58:04,132 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:58:04,133 DEBUG SenderThread:1989491 [sender.py:send():379] send: summary +2025-01-03 22:58:04,136 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:58:04,136 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,136 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:58:04,137 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:58:04,137 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:04,137 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:58:04,137 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,137 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:58:04,137 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:58:04,137 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:04,137 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:04,137 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:58:04,137 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,138 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:58:04,369 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:58:04,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:05,091 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:58:05,091 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:05,091 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:05,092 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:58:05,092 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:05,092 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:58:05,092 INFO SenderThread:1989491 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:58:05,094 INFO SenderThread:1989491 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:58:05,096 INFO SenderThread:1989491 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:58:05,111 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:58:05,111 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:05,111 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:58:05,111 DEBUG SenderThread:1989491 [sender.py:send():379] send: artifact +2025-01-03 22:58:05,369 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:58:05,441 INFO wandb-upload_0:1989491 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2ytwnw8t +2025-01-03 22:58:05,441 INFO wandb-upload_2:1989491 [upload_job.py:push():85] Skipped uploading /tmp/tmp323e5_vb/wandb-job.json +2025-01-03 22:58:05,548 INFO wandb-upload_1:1989491 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7z02pels +2025-01-03 22:58:05,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:06,146 INFO SenderThread:1989491 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'versionIndex': 413}}} +2025-01-03 22:58:06,146 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,146 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:58:06,146 INFO SenderThread:1989491 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:58:06,370 INFO SenderThread:1989491 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files +2025-01-03 22:58:06,370 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml config.yaml +2025-01-03 22:58:06,370 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json wandb-summary.json +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt requirements.txt +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log output.log +2025-01-03 22:58:06,373 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/diff.patch diff.patch +2025-01-03 22:58:06,376 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:58:06,380 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:06,380 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:06,380 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:58:06,380 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,380 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:58:06,380 INFO SenderThread:1989491 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:58:06,572 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt +2025-01-03 22:58:06,576 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:06,576 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:06,587 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml +2025-01-03 22:58:06,596 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:58:06,636 INFO wandb-upload_4:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:58:06,652 INFO wandb-upload_3:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/q24ai29p-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:58:06,853 INFO Thread-11 (_thread_body):1989491 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:58:06,853 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:06,853 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:58:06,854 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,854 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:58:06,854 INFO SenderThread:1989491 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:58:06,855 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:58:06,855 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:06,855 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:58:06,855 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,855 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:58:06,855 INFO SenderThread:1989491 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:58:07,031 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:07,031 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:58:07,031 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:58:07,031 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:07,032 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:58:07,032 DEBUG SenderThread:1989491 [sender.py:send():379] send: final +2025-01-03 22:58:07,032 DEBUG SenderThread:1989491 [sender.py:send():379] send: footer +2025-01-03 22:58:07,032 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:07,032 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:58:07,033 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:07,033 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:07,033 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:58:07,034 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:07,034 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:58:07,034 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:58:07,035 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:58:07,036 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:07,100 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:07,101 INFO MainThread:1989491 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:58:07,101 INFO MainThread:1989491 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:58:07,101 INFO MainThread:1989491 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:58:07,101 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:58:07,102 INFO HandlerThread:1989491 [handler.py:finish():882] shutting down handler +2025-01-03 22:58:08,034 INFO WriterThread:1989491 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/run-l4rsu423.wandb +2025-01-03 22:58:08,101 INFO SenderThread:1989491 [sender.py:finish():1615] shutting down sender +2025-01-03 22:58:08,101 INFO SenderThread:1989491 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:58:08,101 INFO SenderThread:1989491 [file_pusher.py:join():175] waiting for file pusher diff --git a/l4rsu423/wandb/debug.log b/l4rsu423/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4bdad5b245ca4b3b1e04266a4599d59a47ea011d --- /dev/null +++ b/l4rsu423/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:52:29,987 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:52:29,987 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Configure stats pid to 1989297 +2025-01-03 22:52:29,987 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/settings +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug.log +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug-internal.log +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423'} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():619] starting backend +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():623] setting up manager +2025-01-03 22:52:29,992 INFO MainThread:1989297 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:52:29,994 INFO MainThread:1989297 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:52:30,006 INFO MainThread:1989297 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:52:30,014 INFO MainThread:1989297 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:52:30,270 INFO MainThread:1989297 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:52:30,312 INFO MainThread:1989297 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:52:30,313 INFO MainThread:1989297 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:52:30,655 INFO MainThread:1989297 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:52:30,656 INFO MainThread:1989297 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/tb, True +2025-01-03 22:52:36,624 INFO MainThread:1989297 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:58:08,102 WARNING MsgRouterThr:1989297 [router.py:message_loop():77] message_loop has been closed diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH/eval_robot.py b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bb99926705398959aea930c822af069871dc131 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735962749 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/diff.patch b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a2db2594aac2dfee94d7430f5fc4332216ce7fe1 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.37485200169205224, 0.9957208297155283, 0.3695220351480134, 0.9926368444172731, 0.4414144158480513, 0.954582236616294, 0.4180309623643781, 0.934476241382488]]} \ No newline at end of file diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..95ceba014a28931cee06ab7bf757b82a8cdaff21 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[13, 25, 0.52, 0.31563088301531317, 0.7305546998516551, 0.32050228122486435, 0.6970399766766335, 0.34418594840304995, 0.6816850453251819, 0.4089461416188196, 0.6533096581390845]]} \ No newline at end of file diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..adc783ee72d90cd1cf143375ab8177314e772cf6 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ec425df4891952a54d5fb9d467857d25a196a5aca8b03784f4fb0f903ee1c9 +size 1443427 diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1838dc4e3b9820f65682062b91dfaaf139b6270f --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4475b9280ad87e0fbb8278e792cc867599d0604b322f710717190f93ff733c1 +size 1576912 diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac023ebd6f5d74ba35fff8aa971098bb186cb62d --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cf06e2e8c9f28608fdd7cc380d343def13a9bf81c3974a35365fd5df61c59b2 +size 1429048 diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2080f3ea197fc28691477778cb4296084170fd1a --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5735b0e0eaefce2d8e6d528b21bf9dcec1dd65b8aed354e75d02a35abb01e014 +size 1120327 diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..19ec1fa2fcd3e356426ba092d51546463d36d15b --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fd61d3f24849abbec1d0c7d2d1065ce044cc1e0f85a78d84588f2a99b2b4478 +size 1475550 diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..455643bf0530cfcc28b36bc76c2221a77213b1e5 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735962756.625220]: resetting environment +[INFO] [1735962756.629510]: cleared current plan +[INFO] [1735962756.630315]: moving to home +[INFO] [1735962757.632707]: reached home +[INFO] [1735962757.633922]: reset action history +[INFO] [1735962760.349916]: environment reset +Starting episode 0 +[INFO] [1735962760.353885]: resetting environment +[INFO] [1735962760.358309]: cleared current plan +[INFO] [1735962760.359206]: moving to home +[INFO] [1735962761.361769]: reached home +[INFO] [1735962761.363021]: reset action history +[INFO] [1735962764.072159]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735962769.792825]: resetting environment +[INFO] [1735962769.798481]: cleared current plan +[INFO] [1735962769.799386]: moving to home +[INFO] [1735962770.801270]: reached home +[INFO] [1735962770.802531]: reset action history +[INFO] [1735962773.510870]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/episode_rosbags/episode_0_2025-01-03-22-52-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1989297.log +started roslaunch server http://158.130.50.37:41573/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1989602] +process[print_text-2]: started with pid [1989603] +process[rosbag_record-3]: started with pid [1989627] +started bagging! +For topic gripper_width: timestamp difference is 103658694 for nearest: 1735962784528217572 - target: 1735962784424558878 at idx 154 +For topic gripper_width: timestamp difference is 105654533 for nearest: 1735962787128211553 - target: 1735962787022557020 at idx 187 +[INFO] [1735962808.614002]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735962809.369203]: Stored episode 1. +[INFO] [1735962809.369993]: Saving video... +[INFO] [1735962809.567308]: Video saved! +Starting episode 1 +[INFO] [1735962809.579136]: resetting environment +[INFO] [1735962809.588903]: cleared current plan +[INFO] [1735962809.590211]: moving to home +[INFO] [1735962813.196682]: reached home +[INFO] [1735962813.197361]: reset action history +[INFO] [1735962815.909046]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962818.068171]: resetting environment +[INFO] [1735962818.071219]: cleared current plan +[INFO] [1735962818.071572]: moving to home +[INFO] [1735962819.073425]: reached home +[INFO] [1735962819.073947]: reset action history +[INFO] [1735962821.780534]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/episode_rosbags/episode_1_2025-01-03-22-53-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1989297.log +started roslaunch server http://158.130.50.37:33689/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1989751] +process[print_text-5]: started with pid [1989752] +process[rosbag_record-6]: started with pid [1989776] +started bagging! +For topic gripper_width: timestamp difference is 193118499 for nearest: 1735962823261518644 - target: 1735962823068400145 at idx 18 +For topic gripper_width: timestamp difference is -102679827 for nearest: 1735962825961519621 - target: 1735962826064199448 at idx 58 +[INFO] [1735962856.557528]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735962857.322648]: Stored episode 2. +[INFO] [1735962857.326501]: Saving video... +[INFO] [1735962857.673296]: Video saved! +Starting episode 2 +[INFO] [1735962857.720931]: resetting environment +[INFO] [1735962857.758343]: cleared current plan +[INFO] [1735962857.772739]: moving to home +[INFO] [1735962861.376230]: reached home +[INFO] [1735962861.380446]: reset action history +[INFO] [1735962864.162421]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962879.841956]: resetting environment +[INFO] [1735962879.862920]: cleared current plan +[INFO] [1735962879.867993]: moving to home +[INFO] [1735962880.870259]: reached home +[INFO] [1735962880.872097]: reset action history +[INFO] [1735962883.582039]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/episode_rosbags/episode_2_2025-01-03-22-54-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1989297.log +started roslaunch server http://158.130.50.37:46487/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1989894] +process[print_text-8]: started with pid [1989895] +process[rosbag_record-9]: started with pid [1989896] +started bagging! +For topic gripper_width: timestamp difference is -149003385 for nearest: 1735962884161537768 - target: 1735962884310541153 at idx 10 +For topic gripper_width: timestamp difference is 107322894 for nearest: 1735962913961522780 - target: 1735962913854199886 at idx 249 +[INFO] [1735962918.354572]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735962919.169097]: Stored episode 3. +[INFO] [1735962919.178082]: Saving video... +[INFO] [1735962919.444898]: Video saved! +Starting episode 3 +[INFO] [1735962919.505002]: resetting environment +[INFO] [1735962919.520964]: cleared current plan +[INFO] [1735962919.521646]: moving to home +[INFO] [1735962923.124765]: reached home +[INFO] [1735962923.131658]: reset action history +[INFO] [1735962926.065189]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962928.205148]: resetting environment +[INFO] [1735962928.232340]: cleared current plan +[INFO] [1735962928.234849]: moving to home +[INFO] [1735962929.236619]: reached home +[INFO] [1735962929.240167]: reset action history +[INFO] [1735962932.037830]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/episode_rosbags/episode_3_2025-01-03-22-55-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1989297.log +started roslaunch server http://158.130.50.37:45445/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1990033] +process[print_text-11]: started with pid [1990034] +process[rosbag_record-12]: started with pid [1990058] +started bagging! +For topic gripper_width: timestamp difference is -137940344 for nearest: 1735962947061523023 - target: 1735962947199463367 at idx 199 +preempting episode +[INFO] [1735962956.684211]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735962957.504053]: Stored episode 4. +[INFO] [1735962957.509506]: Saving video... +[INFO] [1735962957.835795]: Video saved! +Starting episode 4 +[INFO] [1735962957.898310]: resetting environment +[INFO] [1735962957.930615]: cleared current plan +[INFO] [1735962957.931617]: moving to home +[INFO] [1735962961.136965]: reached home +[INFO] [1735962961.147990]: reset action history +[INFO] [1735962963.936213]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963009.809831]: resetting environment +[INFO] [1735963009.825675]: cleared current plan +[INFO] [1735963009.825877]: moving to home +[INFO] [1735963010.828331]: reached home +[INFO] [1735963010.830230]: reset action history +[INFO] [1735963013.626199]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/episode_rosbags/episode_4_2025-01-03-22-56-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1989297.log +started roslaunch server http://158.130.50.37:43655/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1990241] +process[print_text-14]: started with pid [1990242] +process[rosbag_record-15]: started with pid [1990243] +started bagging! +For topic gripper_width: timestamp difference is 104842811 for nearest: 1735963033661516338 - target: 1735963033556673527 at idx 249 +[INFO] [1735963048.719832]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735963049.469045]: Stored episode 5. +[INFO] [1735963049.475984]: Saving video... +[INFO] [1735963049.821815]: Video saved! +[INFO] [1735963049.869589]: resetting environment +[INFO] [1735963049.914109]: cleared current plan +[INFO] [1735963049.915010]: moving to home +[INFO] [1735963053.524964]: reached home +[INFO] [1735963053.530454]: reset action history +[INFO] [1735963056.333358]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-metadata.json b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..52ce9b9b275da56354fe4c3c1552ac57289f9fcf --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:52:30.338917", + "startedAt": "2025-01-04T03:52:29.985396", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.9480000000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.24, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.364, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 534.1734809875488 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6b33370016244ce6d9254faf04d870a98ae30f98 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "81ec425df4891952a54d5fb9d467857d25a196a5aca8b03784f4fb0f903ee1c9", "size": 1443427, "path": "media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4"}, "global_step": 626, "_timestamp": 1735963060.355065, "_runtime": 310.3608829975128, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "e4475b9280ad87e0fbb8278e792cc867599d0604b322f710717190f93ff733c1", "size": 1576912, "path": "media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4"}, "eval/num_success": 4.0, "episode": 25.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.3748520016670227, "eval/success_rate_ub_0.01": 0.9957208037376404, "eval/success_rate_lb_0.025": 0.3695220351219177, "eval/success_rate_ub_0.025": 0.9926368594169617, "eval/success_rate_lb_0.05": 0.44141441583633423, "eval/success_rate_ub_0.05": 0.9545822143554688, "eval/success_rate_lb_0.1": 0.4180309772491455, "eval/success_rate_ub_0.1": 0.9344762563705444, "eval/2_eval": {"_type": "video-file", "sha256": "7cf06e2e8c9f28608fdd7cc380d343def13a9bf81c3974a35365fd5df61c59b2", "size": 1429048, "path": "media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "5735b0e0eaefce2d8e6d528b21bf9dcec1dd65b8aed354e75d02a35abb01e014", "size": 1120327, "path": "media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "6fd61d3f24849abbec1d0c7d2d1065ce044cc1e0f85a78d84588f2a99b2b4478", "size": 1475550, "path": "media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "8490adedbf62c026e30853c69820c3dd6a1c7da4ee481eab9e8dcf125e285d47", "size": 397, "artifact_path": "wandb-client-artifact://q7nev34veo71uup1c8qhgruhislr1snkaxq24e32jushyyujmafkmf0jv0eytyskdg4h3swn9gf2rosuaqz8d5oggj3kzgnr08vg0yxrzaz6rootr85p5nf5tmdzj99w/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://sac6vs0zpjivny5m0vuju9h0nf3jybpwfwna4jqqys4tix0y50w3ankbl6wjhlyxcbedxx6zrd9v9e121mxcl5e1tohe9tz1hmg0n4g7n51tlnvuhzph6t2urwqugcpt:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 13, "total_num_episodes": 25, "total_success_rate": 0.52, "total_success_rate_lb_0.01": 0.31563088301531317, "total_success_rate_ub_0.01": 0.7305546998516551, "total_success_rate_lb_0.025": 0.32050228122486435, "total_success_rate_ub_0.025": 0.6970399766766335, "total_success_rate_lb_0.05": 0.34418594840304995, "total_success_rate_ub_0.05": 0.6816850453251819, "total_success_rate_lb_0.1": 0.4089461416188196, "total_success_rate_ub_0.1": 0.6533096581390845, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2c9f95e7366fe599681b03bbb56b8515b339e85eacbba0e8c2b966460cbb2911", "size": 509, "artifact_path": "wandb-client-artifact://2jb0y15l63u37sjn3qemk8oxgvf6hnmqlsbjenro0sc3fc2lpbqgpy2xiwk2vw54j9krg38cqbk8z8w3fqmffkeegmztfrweamwml6tp8evnws6ab4javcmq447scxc5/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://bxba84buiqqspkyv1htjnla9nom2wbizsk4tmehlnhljtn6r4jptn5nvfrteqs6724j4gzty0kqjex5thakl0u7gasbfc2tek4l0suiznz3ir4g1r6yo8h495jdu5oqf:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 327}, "eval/total_success_rate": 0.5199999809265137, "eval/total_success_rate_lb_0.01": 0.31563088297843933, "eval/total_success_rate_ub_0.01": 0.7305546998977661, "eval/total_success_rate_lb_0.025": 0.32050228118896484, "eval/total_success_rate_ub_0.025": 0.6970399618148804, "eval/total_success_rate_lb_0.05": 0.3441859483718872, "eval/total_success_rate_ub_0.05": 0.6816850304603577, "eval/total_success_rate_lb_0.1": 0.40894615650177, "eval/total_success_rate_ub_0.1": 0.6533096432685852} \ No newline at end of file diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug-internal.log b/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7f3a2f6581bef9cca7aa6dfe0f4b28eabdcfc9f8 --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug-internal.log @@ -0,0 +1,853 @@ +2025-01-03 22:52:29,994 INFO StreamThr :1989491 [internal.py:wandb_internal():85] W&B internal server running at pid: 1989491, started at: 2025-01-03 22:52:29.993802 +2025-01-03 22:52:29,995 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:52:30,007 INFO WriterThread:1989491 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/run-l4rsu423.wandb +2025-01-03 22:52:30,007 DEBUG SenderThread:1989491 [sender.py:send():379] send: header +2025-01-03 22:52:30,016 DEBUG SenderThread:1989491 [sender.py:send():379] send: run +2025-01-03 22:52:30,256 INFO SenderThread:1989491 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files +2025-01-03 22:52:30,257 INFO SenderThread:1989491 [sender.py:_start_run_threads():1188] run started: l4rsu423 with start time 1735962749.994182 +2025-01-03 22:52:30,271 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:52:30,271 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:52:30,319 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:52:30,324 DEBUG HandlerThread:1989491 [system_info.py:__init__():26] System info init +2025-01-03 22:52:30,324 DEBUG HandlerThread:1989491 [system_info.py:__init__():41] System info init done +2025-01-03 22:52:30,324 INFO HandlerThread:1989491 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:52:30,324 INFO SystemMonitor:1989491 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:52:30,324 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:52:30,325 INFO HandlerThread:1989491 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:52:30,325 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:52:30,325 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:52:30,326 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:52:30,327 INFO SystemMonitor:1989491 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:52:30,338 DEBUG HandlerThread:1989491 [system_info.py:probe():152] Probing system +2025-01-03 22:52:30,341 DEBUG HandlerThread:1989491 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_info.py:probe():200] Probing system done +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:52:30.338917', 'startedAt': '2025-01-04T03:52:29.985396', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.9480000000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.24, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.364, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 534.1734809875488}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:52:30,347 INFO HandlerThread:1989491 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:52:30,347 INFO HandlerThread:1989491 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:52:30,347 DEBUG HandlerThread:1989491 [system_info.py:_save_code():44] Saving code +2025-01-03 22:52:30,348 DEBUG HandlerThread:1989491 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:52:30,348 DEBUG HandlerThread:1989491 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:52:30,410 DEBUG HandlerThread:1989491 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:52:30,410 INFO HandlerThread:1989491 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:52:30,413 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:52:30,413 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:52:30,413 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:52:30,414 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:52:30,652 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:52:30,652 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:52:30,654 DEBUG SenderThread:1989491 [sender.py:send():379] send: telemetry +2025-01-03 22:52:30,684 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/0urikxzw-diff.patch +2025-01-03 22:52:30,686 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/ihtde9je-code/FISH/eval_robot.py +2025-01-03 22:52:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:52:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:30,697 INFO HandlerThread:1989491 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/tb" + save: true +} +control { + relay_id: "ab0a36204cd14eae8020fc5f5ed010c6" +} +_info { + stream_id: "l4rsu423" +} + +2025-01-03 22:52:30,697 DEBUG HandlerThread:1989491 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:52:30,698 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:52:30,704 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/t7zobwhm-wandb-metadata.json +2025-01-03 22:52:30,752 DEBUG SenderThread:1989491 [sender.py:send():379] send: tbrecord +2025-01-03 22:52:30,752 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:52:30,752 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 with policy live +2025-01-03 22:52:30,939 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/2d936hl5-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:52:31,258 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/diff.patch +2025-01-03 22:52:31,258 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt +2025-01-03 22:52:31,258 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH/eval_robot.py +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-metadata.json +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH +2025-01-03 22:52:31,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code +2025-01-03 22:52:31,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:32,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:33,257 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:33,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:34,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:35,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:35,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:36,625 DEBUG SenderThread:1989491 [sender.py:send():379] send: config +2025-01-03 22:52:36,669 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:37,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:38,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:39,259 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:39,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:40,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:40,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:41,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:42,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:43,260 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:43,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:44,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:45,261 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:45,652 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:52:45,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:52:45,693 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:45,729 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:46,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:47,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:48,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:49,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:50,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:50,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:51,263 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:51,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:52,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:53,263 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:53,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:54,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:55,264 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:52:55,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:56,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:52:56,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:57,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:58,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:52:59,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:00,652 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:00,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:00,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:01,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:01,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:02,266 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml +2025-01-03 22:53:02,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:03,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:04,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:05,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:06,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:07,132 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:07,267 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:07,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:08,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:09,268 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:09,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:10,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:11,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:12,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:12,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:13,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:14,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:15,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:15,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:15,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:16,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:17,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:17,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:18,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:19,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:20,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:21,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:22,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:23,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:23,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:24,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:25,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:26,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:27,662 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:28,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:28,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:29,577 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:53:29,577 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 with policy now +2025-01-03 22:53:29,578 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:53:29,578 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:53:29,579 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:53:29,580 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:53:29,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:29,838 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/3w8ct9ld-media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:53:30,273 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media +2025-01-03 22:53:30,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:53:30,327 DEBUG SystemMonitor:1989491 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:53:30,328 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:53:30,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:30,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:31,274 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:31,304 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/u2mbzarq-media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:53:31,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:32,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:33,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:34,237 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:34,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:35,275 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:35,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:36,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:37,276 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:37,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:38,276 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:53:38,513 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/wxznogyx-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:53:38,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:39,277 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:39,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:40,121 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:40,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:41,277 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:41,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:42,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:43,278 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:43,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:44,664 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:45,279 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:45,357 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:45,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:53:45,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:53:45,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:46,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:47,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:48,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:49,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:50,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:50,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:51,280 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:53:51,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:52,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:53,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:54,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:55,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:53:55,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:56,666 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:57,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:58,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:53:59,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:00,331 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:54:00,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:00,653 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:00,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:00,734 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:01,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:02,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:03,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:04,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:05,667 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:06,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:06,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:07,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:08,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:09,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:10,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:11,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:11,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:12,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:13,668 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:14,669 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:15,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:15,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:15,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:16,669 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:17,326 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:17,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:17,714 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:54:17,728 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 with policy now +2025-01-03 22:54:17,757 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:54:17,765 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:54:17,765 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:54:17,772 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:54:17,990 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/u77yxz4q-media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:54:18,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:54:18,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:54:18,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:54:18,548 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/n9afql6n-media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:54:18,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:19,289 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:19,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:20,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:21,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:22,421 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:22,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:23,291 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:23,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:24,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:25,291 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:25,670 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:26,670 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:27,657 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:27,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:28,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:29,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:30,329 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:54:30,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:30,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:30,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:31,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:32,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:32,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:33,672 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:34,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:35,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:36,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:37,673 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:38,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:38,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:39,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:40,125 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:54:40,125 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:54:40,126 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:54:40,296 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:54:40,296 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:54:40,529 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/6qenl5r4-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:54:40,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:41,296 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:41,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:42,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:43,297 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:43,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:43,742 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:44,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:45,298 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:45,653 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:54:45,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:54:45,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:46,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:47,299 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:54:47,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:48,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:48,764 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:49,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:50,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:51,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:52,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:53,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:54,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:54,676 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:55,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:56,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:57,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:58,677 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:54:59,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:54:59,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:00,329 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:55:00,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:00,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:00,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:01,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:02,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:03,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:04,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:04,783 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:05,679 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:06,680 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:07,680 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:08,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:09,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:10,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:10,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:11,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:12,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:13,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:14,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:15,308 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:15,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:15,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:15,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:15,715 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:16,681 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:17,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:18,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:19,309 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:19,500 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:55:19,501 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 with policy now +2025-01-03 22:55:19,501 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:55:19,502 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:55:19,502 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:55:19,512 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:55:19,689 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:19,744 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/hza77wkz-media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:55:20,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:55:20,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:55:20,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:55:20,568 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/pzwu3bx7-media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:55:20,687 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:21,310 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:21,663 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:21,699 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:22,694 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:23,702 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:24,696 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:25,312 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:25,695 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:26,695 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:27,109 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:27,312 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:27,697 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:28,313 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:55:28,699 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:28,964 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:55:28,964 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:55:28,964 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:55:29,313 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:29,313 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:55:29,701 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:30,330 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:55:30,656 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:30,657 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:30,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:31,314 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:31,720 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:32,208 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:32,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:33,314 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:33,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:34,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:35,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:36,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:37,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:37,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:38,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:39,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:40,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:41,703 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:42,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:43,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:43,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:44,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:45,654 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:55:45,654 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:55:45,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:46,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:47,704 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:48,705 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:49,320 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:49,357 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:49,705 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:50,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:51,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:52,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:53,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:54,659 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:55:54,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:55,706 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:56,717 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:57,729 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:57,892 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:55:57,892 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 with policy now +2025-01-03 22:55:57,895 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:55:57,895 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:55:57,895 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:55:57,896 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:55:58,143 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/fki19acd-media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:55:58,323 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:55:58,324 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:55:58,324 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:55:58,573 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/a36ouylc-media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:55:58,736 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:59,324 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:55:59,730 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:55:59,933 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:00,330 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:56:00,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:00,661 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:00,728 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:01,732 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:02,733 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:03,325 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:03,739 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:04,738 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:04,977 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:05,326 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:05,738 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:06,740 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:07,749 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:08,751 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:09,751 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:10,665 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:10,750 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:11,756 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:12,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:13,750 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:14,751 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:15,655 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:15,655 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:15,754 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:15,761 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:16,756 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:17,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:18,755 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:19,757 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:20,758 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:21,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:21,759 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:22,760 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:23,759 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:24,756 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:25,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:26,678 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:26,760 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:27,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:28,761 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:29,764 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:30,330 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:56:30,671 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:30,671 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:30,766 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:31,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:31,769 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:32,767 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:33,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:34,766 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:35,772 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:36,771 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:37,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:37,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:38,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:39,781 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:40,776 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:41,780 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:42,774 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:43,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:43,779 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:44,777 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:45,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:56:45,660 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:56:45,779 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:46,777 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:47,779 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:48,748 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:48,777 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:49,821 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:50,061 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:56:50,061 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:56:50,062 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:56:50,341 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:56:50,341 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:56:50,579 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/ak3oaw6t-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:56:50,791 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:51,341 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:51,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:52,791 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:53,342 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:53,782 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:53,926 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:54,782 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:55,342 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:56:55,782 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:56,783 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:57,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:58,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:56:59,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:56:59,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:00,331 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:57:00,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:00,658 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:00,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:01,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:02,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:03,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:04,734 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:04,786 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:05,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:06,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:07,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:08,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:09,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:10,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:10,787 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:11,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:12,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:13,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:14,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:15,350 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:15,658 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:15,658 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:15,729 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:15,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:16,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:17,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:18,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:19,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:20,788 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:21,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:21,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:22,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:23,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:24,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:25,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:26,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:27,674 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:27,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:28,789 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:29,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:29,869 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:57:29,869 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:57:29,870 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 with policy now +2025-01-03 22:57:29,870 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:29,870 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:29,871 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:30,135 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/5xw1l5rs-media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:57:30,332 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:57:30,355 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:30,356 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:57:30,356 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval +2025-01-03 22:57:30,628 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/1w30f7g5-media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:57:30,675 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:30,675 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:30,796 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:31,356 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:31,801 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:32,797 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:32,832 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:33,798 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:34,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:35,357 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:35,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:36,809 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:37,358 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:37,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:38,377 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:38,815 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:39,811 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:39,916 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:57:39,916 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:57:40,049 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:57:40,359 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:57:40,359 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:57:40,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media +2025-01-03 22:57:40,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval +2025-01-03 22:57:40,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table +2025-01-03 22:57:40,375 INFO wandb-upload_2:1989491 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp49ty0kqa +2025-01-03 22:57:40,434 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:57:40,442 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:57:40,542 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/rtxoxcw8-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:57:40,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:40,833 INFO SenderThread:1989491 [sender.py:send_request_log_artifact():1525] logged artifact run-l4rsu423-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzMzMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MjUzNA==', 'latestArtifact': None}} +2025-01-03 22:57:40,833 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:57:40,833 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-03 22:57:40,833 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:40,833 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,834 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,834 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:40,834 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,838 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,838 DEBUG SenderThread:1989491 [sender.py:send():379] send: telemetry +2025-01-03 22:57:40,838 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,839 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,839 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,839 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,839 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,840 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,840 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,840 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,840 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,841 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,841 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,841 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,841 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,842 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,842 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,842 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,842 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,843 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,843 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,843 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,843 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:40,844 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:40,844 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:57:41,078 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/3ap8bgdi-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:57:41,359 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:57:41,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval +2025-01-03 22:57:41,379 INFO wandb-upload_0:1989491 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnqrwh20p +2025-01-03 22:57:41,581 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/nbblimeg-media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:57:41,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:41,815 INFO SenderThread:1989491 [sender.py:send_request_log_artifact():1525] logged artifact run-l4rsu423-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzM0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MjUzNQ==', 'latestArtifact': None}} +2025-01-03 22:57:41,815 DEBUG SenderThread:1989491 [sender.py:send():379] send: files +2025-01-03 22:57:41,815 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json with policy now +2025-01-03 22:57:41,815 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:57:41,815 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:57:41,816 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:42,085 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/buhdm5bf-media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:57:42,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:42,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:43,360 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:57:43,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:43,816 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:44,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:45,660 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:57:45,660 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:57:45,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:46,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:47,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:48,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:49,804 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:49,812 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:50,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:51,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:52,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:53,805 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:54,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:55,682 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:57:55,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:56,365 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml +2025-01-03 22:57:56,806 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:57:57,574 DEBUG SenderThread:1989491 [sender.py:send():379] send: exit +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:send_exit():588] handling runtime: 327 +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:57:57,574 INFO SenderThread:1989491 [sender.py:send_exit():594] send defer +2025-01-03 22:57:57,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:57:57,575 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:57:57,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:57:57,575 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:57:57,575 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:57:57,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:57:57,575 INFO HandlerThread:1989491 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:57:57,576 DEBUG SystemMonitor:1989491 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:57:57,576 DEBUG SystemMonitor:1989491 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:57:57,576 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:57:57,576 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:57:57,629 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:57:57,630 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:57:57,630 INFO HandlerThread:1989491 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:57:57,630 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,630 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:57:57,630 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:57:57,630 DEBUG SenderThread:1989491 [sender.py:send():379] send: stats +2025-01-03 22:57:57,631 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,631 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:57:57,631 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:57:57,631 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:57:57,631 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:57:57,631 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:57:57,631 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:57:58,366 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:57:59,367 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:58:04,128 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:04,128 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:04,128 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:58:04,130 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,130 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:58:04,130 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:58:04,130 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:04,130 DEBUG SenderThread:1989491 [sender.py:send():379] send: history +2025-01-03 22:58:04,130 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:04,131 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:58:04,131 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:58:04,132 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:58:04,133 DEBUG SenderThread:1989491 [sender.py:send():379] send: summary +2025-01-03 22:58:04,136 INFO SenderThread:1989491 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:58:04,136 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,136 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:58:04,137 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:58:04,137 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:04,137 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:58:04,137 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,137 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:58:04,137 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:58:04,137 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:58:04,137 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:04,137 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:58:04,137 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:04,138 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:58:04,369 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:58:04,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:05,091 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:58:05,091 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:05,091 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:05,092 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:58:05,092 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:05,092 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:58:05,092 INFO SenderThread:1989491 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:58:05,094 INFO SenderThread:1989491 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:58:05,096 INFO SenderThread:1989491 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:58:05,111 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:58:05,111 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:05,111 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:58:05,111 DEBUG SenderThread:1989491 [sender.py:send():379] send: artifact +2025-01-03 22:58:05,369 INFO Thread-12 :1989491 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:58:05,441 INFO wandb-upload_0:1989491 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2ytwnw8t +2025-01-03 22:58:05,441 INFO wandb-upload_2:1989491 [upload_job.py:push():85] Skipped uploading /tmp/tmp323e5_vb/wandb-job.json +2025-01-03 22:58:05,548 INFO wandb-upload_1:1989491 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7z02pels +2025-01-03 22:58:05,575 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:06,146 INFO SenderThread:1989491 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'versionIndex': 413}}} +2025-01-03 22:58:06,146 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,146 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:58:06,146 INFO SenderThread:1989491 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:58:06,370 INFO SenderThread:1989491 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files +2025-01-03 22:58:06,370 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml config.yaml +2025-01-03 22:58:06,370 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json wandb-summary.json +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt requirements.txt +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:58:06,371 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log output.log +2025-01-03 22:58:06,373 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/diff.patch diff.patch +2025-01-03 22:58:06,376 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 media/videos/eval/2_eval_2_7cf06e2e8c9f28608fdd.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 media/videos/eval/0_eval_0_81ec425df4891952a54d.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 media/videos/eval/3_eval_3_5735b0e0eaefce2d8e6d.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 media/videos/eval/4_eval_4_6fd61d3f24849abbec1d.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 media/videos/eval/1_eval_1_e4475b9280ad87e0fbb8.mp4 +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json media/table/eval/total_success_rate_ci_6_2c9f95e7366fe599681b.table.json +2025-01-03 22:58:06,379 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:58:06,380 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:06,380 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:06,380 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:58:06,380 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,380 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:58:06,380 INFO SenderThread:1989491 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:58:06,572 INFO wandb-upload_1:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/requirements.txt +2025-01-03 22:58:06,576 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:06,576 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:06,587 INFO wandb-upload_0:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/config.yaml +2025-01-03 22:58:06,596 INFO wandb-upload_2:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/wandb-summary.json +2025-01-03 22:58:06,636 INFO wandb-upload_4:1989491 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/files/output.log +2025-01-03 22:58:06,652 INFO wandb-upload_3:1989491 [upload_job.py:push():130] Uploaded file /tmp/tmpsmz8w8adwandb/q24ai29p-events.out.tfevents.1735962750.leonmkim-ROG-Strix-G15CS-G15CS.1989297.0 +2025-01-03 22:58:06,853 INFO Thread-11 (_thread_body):1989491 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:58:06,853 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:06,853 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:58:06,854 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,854 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:58:06,854 INFO SenderThread:1989491 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:58:06,855 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:58:06,855 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:06,855 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:58:06,855 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:06,855 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:58:06,855 INFO SenderThread:1989491 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:58:07,031 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:07,031 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:58:07,031 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:58:07,031 INFO SenderThread:1989491 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:58:07,031 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:58:07,032 INFO HandlerThread:1989491 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:58:07,032 DEBUG SenderThread:1989491 [sender.py:send():379] send: final +2025-01-03 22:58:07,032 DEBUG SenderThread:1989491 [sender.py:send():379] send: footer +2025-01-03 22:58:07,032 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: defer +2025-01-03 22:58:07,032 INFO SenderThread:1989491 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:58:07,033 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:07,033 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:07,033 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:58:07,034 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:58:07,034 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:58:07,034 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:58:07,035 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:58:07,036 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:58:07,100 DEBUG SenderThread:1989491 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:58:07,101 INFO MainThread:1989491 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:58:07,101 INFO MainThread:1989491 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:58:07,101 INFO MainThread:1989491 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:58:07,101 DEBUG HandlerThread:1989491 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:58:07,102 INFO HandlerThread:1989491 [handler.py:finish():882] shutting down handler +2025-01-03 22:58:08,034 INFO WriterThread:1989491 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/run-l4rsu423.wandb +2025-01-03 22:58:08,101 INFO SenderThread:1989491 [sender.py:finish():1615] shutting down sender +2025-01-03 22:58:08,101 INFO SenderThread:1989491 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:58:08,101 INFO SenderThread:1989491 [file_pusher.py:join():175] waiting for file pusher diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug.log b/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4bdad5b245ca4b3b1e04266a4599d59a47ea011d --- /dev/null +++ b/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:52:29,987 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:52:29,987 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Configure stats pid to 1989297 +2025-01-03 22:52:29,987 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/settings +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug.log +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/wandb/run-20250103_225229-l4rsu423/logs/debug-internal.log +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423'} +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():619] starting backend +2025-01-03 22:52:29,988 INFO MainThread:1989297 [wandb_init.py:init():623] setting up manager +2025-01-03 22:52:29,992 INFO MainThread:1989297 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:52:29,994 INFO MainThread:1989297 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:52:30,006 INFO MainThread:1989297 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:52:30,014 INFO MainThread:1989297 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:52:30,270 INFO MainThread:1989297 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:52:30,312 INFO MainThread:1989297 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:52:30,313 INFO MainThread:1989297 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:52:30,653 INFO MainThread:1989297 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:52:30,655 INFO MainThread:1989297 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:52:30,656 INFO MainThread:1989297 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/l4rsu423/tb, True +2025-01-03 22:52:36,624 INFO MainThread:1989297 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:58:08,102 WARNING MsgRouterThr:1989297 [router.py:message_loop():77] message_loop has been closed diff --git a/l4rsu423/wandb/run-20250103_225229-l4rsu423/run-l4rsu423.wandb b/l4rsu423/wandb/run-20250103_225229-l4rsu423/run-l4rsu423.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c9500a8b126c38e6b054bb2aeeafa1e49fddbd17 Binary files /dev/null and b/l4rsu423/wandb/run-20250103_225229-l4rsu423/run-l4rsu423.wandb differ diff --git a/nl3lmlc3/episode_rosbags/cam_tf_world.npy b/nl3lmlc3/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nl3lmlc3/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nl3lmlc3/episode_rosbags/depth_K.npy b/nl3lmlc3/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nl3lmlc3/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nl3lmlc3/eval_robot.log b/nl3lmlc3/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..381e5c807aa9278917d31c66fc5e542ec3674dbb --- /dev/null +++ b/nl3lmlc3/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 21:34:07,032][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:34:07,036][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/pzxh5op7/.hydra/config.yaml b/pzxh5op7/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/pzxh5op7/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/pzxh5op7/.hydra/hydra.yaml b/pzxh5op7/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a69c40d15907018d47e86063ef7f65a9d55d3612 --- /dev/null +++ b/pzxh5op7/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/pzxh5op7/.hydra/overrides.yaml b/pzxh5op7/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/pzxh5op7/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/pzxh5op7/episode_rosbags/aligned_depth_to_color_K.npy b/pzxh5op7/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/pzxh5op7/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/pzxh5op7/episode_rosbags/cam_tf_world.npy b/pzxh5op7/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/pzxh5op7/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/pzxh5op7/episode_rosbags/color_K.npy b/pzxh5op7/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/pzxh5op7/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/pzxh5op7/episode_rosbags/depth_K.npy b/pzxh5op7/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/pzxh5op7/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/pzxh5op7/eval_robot.log b/pzxh5op7/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..3dd22cb51a044551998614b85458e25f40323ef7 --- /dev/null +++ b/pzxh5op7/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:20:42,120][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:20:42,123][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/pzxh5op7/eval_video/0_eval.mp4 b/pzxh5op7/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca24ad3d509f5d52671569fcd69ce856f9c6f5fe --- /dev/null +++ b/pzxh5op7/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edb61c24cfb70619857b3efbc1eebe2088fcfdcc6c4fa7caddd98d283bd92899 +size 1512941 diff --git a/pzxh5op7/eval_video/1_eval.mp4 b/pzxh5op7/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd2a052a2bf052bf32eb769a0ec115e874ebb055 --- /dev/null +++ b/pzxh5op7/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:289145cdda89197ca8dd9bada77b0d945171e15c1e9697a0633b2ae27870e3ef +size 1512599 diff --git a/pzxh5op7/eval_video/2_eval.mp4 b/pzxh5op7/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f182af19ada7af369eb25f1415ae806f9e0a1d5 --- /dev/null +++ b/pzxh5op7/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7358c10e28477967d40b5e5beea6ec947cae475a657017d815d05f005870e93 +size 1574153 diff --git a/pzxh5op7/eval_video/3_eval.mp4 b/pzxh5op7/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da4be8f8f0a1cccbea46bd0177ef3c33fabf9bee --- /dev/null +++ b/pzxh5op7/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b257116aa31f0e2642be6a6f04f34be9b5dfa24a704530001ed46e81baa9357d +size 1561736 diff --git a/pzxh5op7/eval_video/4_eval.mp4 b/pzxh5op7/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..343cc1684fc32105b03c1c3988bc04749d3d9ad0 --- /dev/null +++ b/pzxh5op7/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f785ef3290c03ae127b85b407a95b111505ed474c1045985dfc73e7f1b744ce6 +size 1688174 diff --git a/pzxh5op7/tb/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 b/pzxh5op7/tb/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 new file mode 100644 index 0000000000000000000000000000000000000000..77779cc5fe0b17184cc21ecf3c3fd78457fd438c --- /dev/null +++ b/pzxh5op7/tb/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b0243d21f94d466871af0c18eba1903347f53e191500c5e40280550ab4da745 +size 6539 diff --git a/pzxh5op7/wandb/debug-internal.log b/pzxh5op7/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..064c4ed0c24638192673f8c8411ad7cf0698a28a --- /dev/null +++ b/pzxh5op7/wandb/debug-internal.log @@ -0,0 +1,834 @@ +2025-01-04 22:20:47,833 INFO StreamThr :2035853 [internal.py:wandb_internal():85] W&B internal server running at pid: 2035853, started at: 2025-01-04 22:20:47.832721 +2025-01-04 22:20:47,834 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:20:47,847 INFO WriterThread:2035853 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/run-pzxh5op7.wandb +2025-01-04 22:20:47,847 DEBUG SenderThread:2035853 [sender.py:send():379] send: header +2025-01-04 22:20:47,854 DEBUG SenderThread:2035853 [sender.py:send():379] send: run +2025-01-04 22:20:48,104 INFO SenderThread:2035853 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files +2025-01-04 22:20:48,104 INFO SenderThread:2035853 [sender.py:_start_run_threads():1188] run started: pzxh5op7 with start time 1736047247.833326 +2025-01-04 22:20:48,120 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:20:48,121 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:20:48,195 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:20:48,200 DEBUG HandlerThread:2035853 [system_info.py:__init__():26] System info init +2025-01-04 22:20:48,200 DEBUG HandlerThread:2035853 [system_info.py:__init__():41] System info init done +2025-01-04 22:20:48,200 INFO HandlerThread:2035853 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:20:48,200 INFO SystemMonitor:2035853 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:20:48,200 INFO HandlerThread:2035853 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:20:48,201 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:20:48,202 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:20:48,202 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:20:48,203 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:20:48,203 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:20:48,214 DEBUG HandlerThread:2035853 [system_info.py:probe():152] Probing system +2025-01-04 22:20:48,216 DEBUG HandlerThread:2035853 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_info.py:probe():200] Probing system done +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:20:48.214633', 'startedAt': '2025-01-05T03:20:47.821269', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.004625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.032, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 749.0845489501953}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:20:48,220 INFO HandlerThread:2035853 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:20:48,220 INFO HandlerThread:2035853 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_info.py:_save_code():44] Saving code +2025-01-04 22:20:48,222 DEBUG HandlerThread:2035853 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:20:48,222 DEBUG HandlerThread:2035853 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:20:48,284 DEBUG HandlerThread:2035853 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:20:48,284 INFO HandlerThread:2035853 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:20:48,287 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:20:48,287 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:20:48,288 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:20:48,288 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:20:48,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:20:48,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:20:48,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:20:48,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:20:48,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:48,520 INFO HandlerThread:2035853 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/tb" + save: true +} +control { + relay_id: "cf4338593ed94baa81a5415d41e7825f" +} +_info { + stream_id: "pzxh5op7" +} + +2025-01-04 22:20:48,521 DEBUG HandlerThread:2035853 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:20:48,535 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/vdc39fv3-wandb-metadata.json +2025-01-04 22:20:48,566 INFO wandb-upload_1:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/zya7rh51-code/FISH/eval_robot.py +2025-01-04 22:20:48,594 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/dskr9ok0-diff.patch +2025-01-04 22:20:48,677 DEBUG SenderThread:2035853 [sender.py:send():379] send: telemetry +2025-01-04 22:20:48,678 DEBUG SenderThread:2035853 [sender.py:send():379] send: tbrecord +2025-01-04 22:20:48,679 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:20:48,679 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 with policy live +2025-01-04 22:20:48,894 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/85jw3xng-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:20:49,107 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/diff.patch +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH/eval_robot.py +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-metadata.json +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:20:49,109 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:49,109 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH +2025-01-04 22:20:49,109 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code +2025-01-04 22:20:49,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:50,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:51,105 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:51,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:52,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:53,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:53,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:53,704 DEBUG SenderThread:2035853 [sender.py:send():379] send: config +2025-01-04 22:20:54,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:55,106 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:55,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:56,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:57,106 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:57,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:58,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:58,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:59,107 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:59,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:00,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:01,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:01,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:02,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:03,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:03,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:03,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:03,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:03,632 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:04,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:05,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:06,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:07,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:08,521 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:09,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:09,521 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:10,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:11,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:12,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:13,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:14,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:15,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:15,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:16,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:17,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:18,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:18,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:18,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:19,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:20,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:20,925 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:21,113 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml +2025-01-04 22:21:21,113 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:21,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:22,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:23,114 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:23,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:24,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:25,114 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:25,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:26,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:26,964 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:27,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:28,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:29,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:30,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:31,116 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:31,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:32,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:32,861 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:33,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:33,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:33,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:34,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:35,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:36,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:37,118 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:37,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:38,228 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:38,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:39,118 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:39,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:40,526 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:41,526 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:42,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:43,521 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:43,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:44,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:45,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:46,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:47,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:48,204 DEBUG SystemMonitor:2035853 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:21:48,204 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:21:48,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:48,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:48,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:48,589 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:49,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:50,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:51,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:52,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:53,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:54,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:54,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:55,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:56,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:57,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:58,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:59,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:59,814 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:00,032 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:22:00,032 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 with policy now +2025-01-04 22:22:00,033 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:22:00,034 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:22:00,034 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:22:00,034 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos +2025-01-04 22:22:00,261 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/vhdbrt9c-media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:22:00,522 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/0xwpfcvj-media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:22:00,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:01,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:01,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:02,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:03,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:03,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:03,531 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:04,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:05,125 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:05,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:05,667 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:06,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:07,126 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:07,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:08,127 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:22:08,362 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/bsubc4ah-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:22:08,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:09,127 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:09,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:10,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:11,041 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:11,127 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:11,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:12,531 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:13,128 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:13,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:14,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:15,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:16,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:16,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:17,533 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:18,208 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:22:18,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:18,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:18,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:19,130 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:19,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:20,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:21,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:21,617 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:22,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:23,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:24,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:25,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:26,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:27,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:27,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:28,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:29,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:30,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:31,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:32,526 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:32,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:33,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:33,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:33,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:34,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:35,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:36,535 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:37,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:37,667 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:38,535 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:39,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:40,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:41,538 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:42,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:43,048 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:43,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:44,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:45,138 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:45,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:46,541 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:47,502 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:22:47,503 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 with policy now +2025-01-04 22:22:47,503 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:22:47,504 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:22:47,504 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:22:47,509 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:22:47,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:47,746 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/81gezt0y-media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:22:48,139 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:22:48,139 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:22:48,140 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:22:48,205 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:22:48,206 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:48,408 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/f35c584z-media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:22:48,535 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:48,535 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:48,577 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:49,139 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:49,544 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:50,541 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:51,543 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:52,546 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:53,141 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:53,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:53,548 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:54,550 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:55,142 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:22:55,143 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:55,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:55,966 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:22:55,966 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:22:55,966 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:22:56,143 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:22:56,548 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:57,143 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:57,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:58,533 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:58,550 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:59,550 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:00,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:01,144 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:01,548 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:02,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:03,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:03,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:03,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:03,654 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:04,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:05,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:06,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:07,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:08,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:09,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:09,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:10,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:11,551 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:12,551 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:13,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:14,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:15,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:15,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:16,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:17,553 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:18,206 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:23:18,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:18,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:18,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:19,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:20,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:20,608 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:21,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:22,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:23,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:24,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:25,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:26,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:26,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:27,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:28,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:29,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:30,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:31,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:32,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:32,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:33,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:33,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:33,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:34,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:35,155 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:35,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:36,133 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:23:36,133 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 with policy now +2025-01-04 22:23:36,135 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:23:36,136 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:23:36,136 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:23:36,143 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:23:36,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:23:36,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:23:36,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:23:36,353 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/xh41ftsh-media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:23:36,560 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:36,607 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/et5ojzyg-media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:23:37,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:37,566 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:38,164 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:38,567 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:39,562 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:40,563 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:41,157 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:41,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:42,564 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:43,158 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:43,409 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:43,568 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:44,566 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:45,159 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:45,571 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:46,570 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:47,568 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:48,206 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:23:48,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:48,525 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:48,576 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:48,651 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:49,574 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:50,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:51,577 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:52,581 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:53,576 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:53,718 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:53,820 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:23:53,820 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:23:53,821 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:23:54,162 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:23:54,162 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:23:54,398 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/87tlgr8t-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:23:54,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:55,162 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:55,586 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:56,578 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:57,163 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:57,576 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:58,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:58,792 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:59,163 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:59,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:00,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:01,164 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:01,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:02,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:03,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:03,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:03,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:04,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:04,657 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:05,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:06,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:07,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:08,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:09,167 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:09,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:10,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:10,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:11,581 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:12,581 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:13,582 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:14,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:15,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:15,582 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:16,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:17,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:18,207 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:24:18,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:18,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:18,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:19,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:20,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:20,594 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:21,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:22,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:23,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:24,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:25,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:26,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:26,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:27,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:28,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:29,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:30,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:31,174 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:31,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:31,674 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:32,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:33,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:33,521 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:33,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:34,594 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:34,689 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:24:34,689 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 with policy now +2025-01-04 22:24:34,694 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:24:34,694 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:24:34,694 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:24:34,705 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:24:34,930 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/ab0s6r8z-media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:24:35,175 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:35,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:24:35,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:24:35,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:24:35,442 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/ixmz1iwy-media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:24:35,591 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:36,619 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:36,714 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:37,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:37,607 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:38,602 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:39,177 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:39,600 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:40,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:41,609 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:41,765 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:42,610 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:43,178 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:43,605 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:43,878 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:24:43,878 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:24:43,879 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:24:44,179 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:24:44,179 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:24:44,610 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:45,179 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:45,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:46,608 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:47,180 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:47,253 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:47,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:48,207 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:24:48,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:48,527 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:48,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:49,180 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:49,609 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:50,629 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:51,181 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:51,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:52,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:52,773 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:53,182 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:53,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:54,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:55,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:56,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:57,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:58,531 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:58,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:59,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:00,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:01,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:02,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:03,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:03,521 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:03,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:03,680 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:04,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:05,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:06,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:07,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:08,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:09,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:09,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:10,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:11,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:12,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:13,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:14,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:15,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:15,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:16,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:17,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:18,208 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:25:18,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:18,522 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:18,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:19,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:20,607 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:20,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:21,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:22,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:23,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:24,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:25,635 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:25,987 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:26,267 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:25:26,267 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 with policy now +2025-01-04 22:25:26,276 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:25:26,276 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:26,277 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:26,278 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:26,501 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/j2wtwy25-media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:25:26,616 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:27,193 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:27,193 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:25:27,194 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:25:27,194 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:25:27,473 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/c4uaa5n6-media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:25:27,629 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:28,628 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:29,631 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:30,627 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:31,195 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:31,385 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:31,634 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:32,635 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:33,195 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:33,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:33,525 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:33,636 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:34,639 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:35,636 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:36,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:36,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:37,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:38,197 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:25:38,430 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/zkxgnqhx-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:25:38,648 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:38,867 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:25:38,868 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:25:39,022 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:25:39,197 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval +2025-01-04 22:25:39,373 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/58nt0szu-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:25:39,397 INFO wandb-upload_0:2035853 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4jies8t4 +2025-01-04 22:25:39,500 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:25:39,510 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:25:39,637 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:39,809 INFO SenderThread:2035853 [sender.py:send_request_log_artifact():1525] logged artifact run-pzxh5op7-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5MDY3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MjczMQ==', 'latestArtifact': None}} +2025-01-04 22:25:39,809 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:25:39,809 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 22:25:39,809 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:39,809 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,810 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,810 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:39,810 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,811 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,811 DEBUG SenderThread:2035853 [sender.py:send():379] send: telemetry +2025-01-04 22:25:39,811 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,812 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,813 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,813 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,813 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,813 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,814 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,814 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,814 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,814 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,814 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,815 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,815 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,815 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,815 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,815 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,816 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,816 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,816 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,816 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,817 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,817 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,817 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:25:40,048 INFO wandb-upload_1:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/wrb7lgly-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:25:40,186 INFO wandb-upload_0:2035853 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpizmnphev +2025-01-04 22:25:40,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:25:40,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:25:40,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:25:40,199 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval +2025-01-04 22:25:40,536 INFO SenderThread:2035853 [sender.py:send_request_log_artifact():1525] logged artifact run-pzxh5op7-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5MDY3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MjczMg==', 'latestArtifact': None}} +2025-01-04 22:25:40,536 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:25:40,536 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json with policy now +2025-01-04 22:25:40,537 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:40,537 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:40,537 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:40,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:40,716 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/dxi9kibz-media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:25:40,945 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/ng8ayly3-media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:25:41,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:41,199 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:25:41,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:42,541 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:42,645 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:43,199 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml +2025-01-04 22:25:43,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:44,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:45,641 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:46,641 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:47,641 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:48,208 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:25:48,208 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:48,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:48,523 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:48,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:49,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:50,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:51,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:52,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:53,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:53,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:54,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:55,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:56,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:57,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:58,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:59,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:59,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:00,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:00,833 DEBUG SenderThread:2035853 [sender.py:send():379] send: exit +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_exit():588] handling runtime: 312 +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_exit():594] send defer +2025-01-04 22:26:00,834 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,834 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:26:00,834 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:26:00,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,835 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:26:00,835 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,835 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:26:00,835 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:26:00,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,835 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:26:00,835 INFO HandlerThread:2035853 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:26:00,835 DEBUG SystemMonitor:2035853 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:26:00,835 DEBUG SystemMonitor:2035853 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:26:00,836 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:26:00,836 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:26:00,886 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:26:00,887 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:26:00,887 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:26:00,887 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,887 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:26:00,887 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:26:00,887 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:26:00,887 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,888 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:26:00,888 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,888 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:26:00,888 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:26:00,888 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,888 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:26:01,206 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:26:03,207 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:26:06,950 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:06,950 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:06,950 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:26:06,951 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,951 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:26:06,951 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:26:06,952 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:06,952 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:06,952 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:26:06,952 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:26:06,952 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:26:06,953 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:26:06,953 DEBUG SenderThread:2035853 [sender.py:send():379] send: summary +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:26:06,954 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:26:06,954 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:06,954 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:26:06,954 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:26:06,954 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:06,954 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:06,954 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:26:06,955 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,955 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:26:07,041 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:26:07,041 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:07,041 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:26:07,041 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:07,041 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:26:07,041 INFO SenderThread:2035853 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:26:07,044 INFO SenderThread:2035853 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:26:07,046 INFO SenderThread:2035853 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:26:07,056 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:26:07,056 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:07,057 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:26:07,057 DEBUG SenderThread:2035853 [sender.py:send():379] send: artifact +2025-01-04 22:26:07,209 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:26:07,209 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:26:07,253 INFO SenderThread:2035853 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 22:26:07,253 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:07,253 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:26:07,253 INFO SenderThread:2035853 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:26:07,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml config.yaml +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json wandb-summary.json +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt requirements.txt +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log output.log +2025-01-04 22:26:08,214 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/diff.patch diff.patch +2025-01-04 22:26:08,214 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:26:08,220 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,220 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,220 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:26:08,221 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,221 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:26:08,221 INFO SenderThread:2035853 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:26:08,397 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml +2025-01-04 22:26:08,408 INFO wandb-upload_1:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:26:08,446 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/h5c5bxk7-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:26:08,454 INFO wandb-upload_3:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt +2025-01-04 22:26:08,490 INFO wandb-upload_4:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:26:08,690 INFO Thread-11 (_thread_body):2035853 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:26:08,690 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,691 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:26:08,691 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,691 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:26:08,691 INFO SenderThread:2035853 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:26:08,692 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:26:08,692 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,692 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:26:08,692 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,692 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:26:08,693 INFO SenderThread:2035853 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:26:08,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,874 INFO SenderThread:2035853 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:26:08,874 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:26:08,874 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,875 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,875 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:26:08,875 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,875 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:26:08,875 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:26:08,875 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,875 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:26:08,875 DEBUG SenderThread:2035853 [sender.py:send():379] send: final +2025-01-04 22:26:08,876 DEBUG SenderThread:2035853 [sender.py:send():379] send: footer +2025-01-04 22:26:08,876 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,876 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:26:08,876 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,876 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,876 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:26:08,877 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,877 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:26:08,877 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:26:08,879 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:26:08,879 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:08,933 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,934 INFO MainThread:2035853 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:26:08,934 INFO MainThread:2035853 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:26:08,934 INFO MainThread:2035853 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:26:08,935 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:26:08,935 INFO HandlerThread:2035853 [handler.py:finish():882] shutting down handler +2025-01-04 22:26:09,877 INFO WriterThread:2035853 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/run-pzxh5op7.wandb +2025-01-04 22:26:09,934 INFO SenderThread:2035853 [sender.py:finish():1615] shutting down sender +2025-01-04 22:26:09,934 INFO SenderThread:2035853 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:26:09,934 INFO SenderThread:2035853 [file_pusher.py:join():175] waiting for file pusher diff --git a/pzxh5op7/wandb/debug.log b/pzxh5op7/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0ebde1df55d07fbb8fd2f7052c285bb5cf70ff2d --- /dev/null +++ b/pzxh5op7/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Configure stats pid to 2035658 +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/settings +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug.log +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug-internal.log +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7'} +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():619] starting backend +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():623] setting up manager +2025-01-04 22:20:47,831 INFO MainThread:2035658 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:20:47,833 INFO MainThread:2035658 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:20:47,846 INFO MainThread:2035658 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:20:47,852 INFO MainThread:2035658 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:20:48,120 INFO MainThread:2035658 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:20:48,191 INFO MainThread:2035658 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:20:48,191 INFO MainThread:2035658 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:20:48,518 INFO MainThread:2035658 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:20:48,518 INFO MainThread:2035658 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:20:48,518 INFO MainThread:2035658 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:20:48,519 INFO MainThread:2035658 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:20:48,520 INFO MainThread:2035658 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:20:48,520 INFO MainThread:2035658 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/tb, True +2025-01-04 22:20:53,704 INFO MainThread:2035658 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:26:09,935 WARNING MsgRouterThr:2035658 [router.py:message_loop():77] message_loop has been closed diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH/eval_robot.py b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcd42b65aa3510e5280193fe3f16513edb528d30 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736047247 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/diff.patch b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6a2e4efdfd7f69cebdb8d8cd6126b9cdb7acc6b5 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[26, 55, 0.4727272727272727, 0.33051812652162826, 0.6236080527058386, 0.34042339030119484, 0.5989851802389539, 0.35768740627035633, 0.5845432206819249, 0.3950892873318209, 0.5629910454028033]]} \ No newline at end of file diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca24ad3d509f5d52671569fcd69ce856f9c6f5fe --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edb61c24cfb70619857b3efbc1eebe2088fcfdcc6c4fa7caddd98d283bd92899 +size 1512941 diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd2a052a2bf052bf32eb769a0ec115e874ebb055 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:289145cdda89197ca8dd9bada77b0d945171e15c1e9697a0633b2ae27870e3ef +size 1512599 diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f182af19ada7af369eb25f1415ae806f9e0a1d5 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7358c10e28477967d40b5e5beea6ec947cae475a657017d815d05f005870e93 +size 1574153 diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da4be8f8f0a1cccbea46bd0177ef3c33fabf9bee --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b257116aa31f0e2642be6a6f04f34be9b5dfa24a704530001ed46e81baa9357d +size 1561736 diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..343cc1684fc32105b03c1c3988bc04749d3d9ad0 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f785ef3290c03ae127b85b407a95b111505ed474c1045985dfc73e7f1b744ce6 +size 1688174 diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9bd54ebce00f77e65380f855cd577a4824131117 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736047253.704611]: resetting environment +[INFO] [1736047253.711512]: cleared current plan +[INFO] [1736047253.712688]: moving to home +[INFO] [1736047254.714737]: reached home +[INFO] [1736047254.716160]: reset action history +[INFO] [1736047257.424606]: environment reset +Starting episode 0 +[INFO] [1736047257.430686]: resetting environment +[INFO] [1736047257.438812]: cleared current plan +[INFO] [1736047257.440836]: moving to home +[INFO] [1736047258.443047]: reached home +[INFO] [1736047258.444445]: reset action history +[INFO] [1736047261.154634]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736047279.911844]: resetting environment +[INFO] [1736047279.916507]: cleared current plan +[INFO] [1736047279.917758]: moving to home +[INFO] [1736047280.921890]: reached home +[INFO] [1736047280.923739]: reset action history +[INFO] [1736047283.630768]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/episode_rosbags/episode_0_2025-01-04-22-21-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2035658.log +started roslaunch server http://158.130.50.37:35545/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2035985] +process[print_text-2]: started with pid [2035986] +process[rosbag_record-3]: started with pid [2035987] +started bagging! +For topic gripper_width: timestamp difference is 100156958 for nearest: 1736047289818180973 - target: 1736047289718024015 at idx 85 +For topic gripper_width: timestamp difference is 101999190 for nearest: 1736047295618181851 - target: 1736047295516182661 at idx 163 +For topic gripper_width: timestamp difference is 104090954 for nearest: 1736047298218179251 - target: 1736047298114088297 at idx 199 +[INFO] [1736047319.040919]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736047319.813438]: Stored episode 1. +[INFO] [1736047319.813946]: Saving video... +[INFO] [1736047320.017289]: Video saved! +Starting episode 1 +[INFO] [1736047320.033443]: resetting environment +[INFO] [1736047320.041893]: cleared current plan +[INFO] [1736047320.044660]: moving to home +[INFO] [1736047323.448573]: reached home +[INFO] [1736047323.449127]: reset action history +[INFO] [1736047326.158506]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047327.316193]: resetting environment +[INFO] [1736047327.328051]: cleared current plan +[INFO] [1736047327.328249]: moving to home +[INFO] [1736047328.329265]: reached home +[INFO] [1736047328.329716]: reset action history +[INFO] [1736047331.040295]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/episode_rosbags/episode_1_2025-01-04-22-22-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2035658.log +started roslaunch server http://158.130.50.37:42885/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2036134] +process[print_text-5]: started with pid [2036135] +process[rosbag_record-6]: started with pid [2036136] +started bagging! +For topic gripper_width: timestamp difference is 102058484 for nearest: 1736047336918189122 - target: 1736047336816130638 at idx 89 +For topic gripper_width: timestamp difference is -110715775 for nearest: 1736047362851520868 - target: 1736047362962236643 at idx 249 +[INFO] [1736047366.399442]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736047367.203243]: Stored episode 2. +[INFO] [1736047367.208590]: Saving video... +[INFO] [1736047367.453746]: Video saved! +Starting episode 2 +[INFO] [1736047367.507423]: resetting environment +[INFO] [1736047367.526582]: cleared current plan +[INFO] [1736047367.528218]: moving to home +[INFO] [1736047370.930808]: reached home +[INFO] [1736047370.936372]: reset action history +[INFO] [1736047373.721759]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047374.940766]: resetting environment +[INFO] [1736047374.967442]: cleared current plan +[INFO] [1736047374.968554]: moving to home +[INFO] [1736047375.971654]: reached home +[INFO] [1736047375.973228]: reset action history +[INFO] [1736047378.702173]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/episode_rosbags/episode_2_2025-01-04-22-22-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2035658.log +started roslaunch server http://158.130.50.37:45369/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2036277] +process[print_text-8]: started with pid [2036278] +process[rosbag_record-9]: started with pid [2036279] +started bagging! +For topic gripper_width: timestamp difference is 100262369 for nearest: 1736047413721057619 - target: 1736047413620795250 at idx 249 +[INFO] [1736047415.011935]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736047415.848438]: Stored episode 3. +[INFO] [1736047415.852551]: Saving video... +[INFO] [1736047416.093938]: Video saved! +Starting episode 3 +[INFO] [1736047416.140862]: resetting environment +[INFO] [1736047416.161942]: cleared current plan +[INFO] [1736047416.162316]: moving to home +[INFO] [1736047419.469661]: reached home +[INFO] [1736047419.471226]: reset action history +[INFO] [1736047422.364026]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047433.750739]: resetting environment +[INFO] [1736047433.785048]: cleared current plan +[INFO] [1736047433.789894]: moving to home +[INFO] [1736047434.791836]: reached home +[INFO] [1736047434.793199]: reset action history +[INFO] [1736047437.522115]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/episode_rosbags/episode_3_2025-01-04-22-23-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2035658.log +started roslaunch server http://158.130.50.37:34523/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2036420] +process[print_text-11]: started with pid [2036421] +process[rosbag_record-12]: started with pid [2036422] +started bagging! +For topic gripper_width: timestamp difference is -108547140 for nearest: 1736047447551512312 - target: 1736047447660059452 at idx 137 +For topic gripper_width: timestamp difference is 108334613 for nearest: 1736047468618178916 - target: 1736047468509844303 at idx 249 +[INFO] [1736047473.574135]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736047474.402272]: Stored episode 4. +[INFO] [1736047474.423164]: Saving video... +[INFO] [1736047474.644082]: Video saved! +Starting episode 4 +[INFO] [1736047474.697449]: resetting environment +[INFO] [1736047474.713409]: cleared current plan +[INFO] [1736047474.714170]: moving to home +[INFO] [1736047477.919934]: reached home +[INFO] [1736047477.924057]: reset action history +[INFO] [1736047480.721720]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047483.429510]: resetting environment +[INFO] [1736047483.443978]: cleared current plan +[INFO] [1736047483.444232]: moving to home +[INFO] [1736047484.449293]: reached home +[INFO] [1736047484.452609]: reset action history +[INFO] [1736047487.253408]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/episode_rosbags/episode_4_2025-01-04-22-24-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2035658.log +started roslaunch server http://158.130.50.37:40323/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2036573] +process[print_text-14]: started with pid [2036574] +process[rosbag_record-15]: started with pid [2036575] +started bagging! +For topic gripper_width: timestamp difference is -185286676 for nearest: 1736047491451513136 - target: 1736047491636799812 at idx 55 +[INFO] [1736047525.220476]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736047525.985122]: Stored episode 5. +[INFO] [1736047525.990086]: Saving video... +[INFO] [1736047526.220414]: Video saved! +[INFO] [1736047526.276838]: resetting environment +[INFO] [1736047526.326137]: cleared current plan +[INFO] [1736047526.335736]: moving to home +[INFO] [1736047529.342891]: reached home +[INFO] [1736047529.349992]: reset action history +[INFO] [1736047532.135423]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-metadata.json b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2626b93f1fa84bcb8ceecee59a52cbf7edb78e09 --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:20:48.214633", + "startedAt": "2025-01-05T03:20:47.821269", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3750.004625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.032, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 749.0845489501953 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2fdf043d2d7c6fe34aa73af4980ed514408f476a --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "edb61c24cfb70619857b3efbc1eebe2088fcfdcc6c4fa7caddd98d283bd92899", "size": 1512941, "path": "media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4"}, "global_step": 665, "_timestamp": 1736047539.3719983, "_runtime": 291.538672208786, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "289145cdda89197ca8dd9bada77b0d945171e15c1e9697a0633b2ae27870e3ef", "size": 1512599, "path": "media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4"}, "eval/num_success": 0.0, "episode": 55.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "f7358c10e28477967d40b5e5beea6ec947cae475a657017d815d05f005870e93", "size": 1574153, "path": "media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "b257116aa31f0e2642be6a6f04f34be9b5dfa24a704530001ed46e81baa9357d", "size": 1561736, "path": "media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "f785ef3290c03ae127b85b407a95b111505ed474c1045985dfc73e7f1b744ce6", "size": 1688174, "path": "media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://u44iyr01aj5t188l5wqt2rlx37tfcxd5gm58s0v4y4ua40llco7b0oef25hvv8nkkchcg8uewl0j6cxpzjjphnv70rqs8whzrh1xr7b3m7zxuult3e0n8er5bd8cxpcv/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://cfn1ex8f2lmvmqa55ad0aaelms7cica6zs9wohrbqa0h4a6o3k509alaivwfu6zj80badx2p5a5s4koj25y6kj9a9ny0zb61ucg025gmxweh7yphxwbafzvzzrhaklsl:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 26, "total_num_episodes": 55, "total_success_rate": 0.4727272727272727, "total_success_rate_lb_0.01": 0.33051812652162826, "total_success_rate_ub_0.01": 0.6236080527058386, "total_success_rate_lb_0.025": 0.34042339030119484, "total_success_rate_ub_0.025": 0.5989851802389539, "total_success_rate_lb_0.05": 0.35768740627035633, "total_success_rate_ub_0.05": 0.5845432206819249, "total_success_rate_lb_0.1": 0.3950892873318209, "total_success_rate_ub_0.1": 0.5629910454028033, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "82442c864ae09c6d7e48f7bb88856e98895ab524c4101a66cd6c77e6fc2e77ea", "size": 523, "artifact_path": "wandb-client-artifact://rtj83cvuw0titm73ga9aqhbzl9g7d3hip2qj80esx2iv5uqg189rupk88cyq7ojckw2j8z97c68saguzwuklyqiem5unkmfktd8k3esw6r8oepsf1k7rdtokw8uf23th/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://aw3z8g2oegl5wc6xlgm0ppbnaqfddwlp6tmqxfsw8c6d0ajx4sqsunrlmy3yahlijamzb359oz8qldj852w0yn0eww4xq0l68kbrik5mzdj79ksn7zha2jxdqkzv9l4b:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 312}, "eval/total_success_rate": 0.4727272689342499, "eval/total_success_rate_lb_0.01": 0.33051812648773193, "eval/total_success_rate_ub_0.01": 0.6236080527305603, "eval/total_success_rate_lb_0.025": 0.3404234051704407, "eval/total_success_rate_ub_0.025": 0.5989851951599121, "eval/total_success_rate_lb_0.05": 0.35768741369247437, "eval/total_success_rate_ub_0.05": 0.5845432281494141, "eval/total_success_rate_lb_0.1": 0.3950892984867096, "eval/total_success_rate_ub_0.1": 0.5629910230636597} \ No newline at end of file diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug-internal.log b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..064c4ed0c24638192673f8c8411ad7cf0698a28a --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug-internal.log @@ -0,0 +1,834 @@ +2025-01-04 22:20:47,833 INFO StreamThr :2035853 [internal.py:wandb_internal():85] W&B internal server running at pid: 2035853, started at: 2025-01-04 22:20:47.832721 +2025-01-04 22:20:47,834 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:20:47,847 INFO WriterThread:2035853 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/run-pzxh5op7.wandb +2025-01-04 22:20:47,847 DEBUG SenderThread:2035853 [sender.py:send():379] send: header +2025-01-04 22:20:47,854 DEBUG SenderThread:2035853 [sender.py:send():379] send: run +2025-01-04 22:20:48,104 INFO SenderThread:2035853 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files +2025-01-04 22:20:48,104 INFO SenderThread:2035853 [sender.py:_start_run_threads():1188] run started: pzxh5op7 with start time 1736047247.833326 +2025-01-04 22:20:48,120 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:20:48,121 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:20:48,195 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:20:48,200 DEBUG HandlerThread:2035853 [system_info.py:__init__():26] System info init +2025-01-04 22:20:48,200 DEBUG HandlerThread:2035853 [system_info.py:__init__():41] System info init done +2025-01-04 22:20:48,200 INFO HandlerThread:2035853 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:20:48,200 INFO SystemMonitor:2035853 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:20:48,200 INFO HandlerThread:2035853 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:20:48,201 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:20:48,202 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:20:48,202 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:20:48,203 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:20:48,203 INFO SystemMonitor:2035853 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:20:48,214 DEBUG HandlerThread:2035853 [system_info.py:probe():152] Probing system +2025-01-04 22:20:48,216 DEBUG HandlerThread:2035853 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_info.py:probe():200] Probing system done +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:20:48.214633', 'startedAt': '2025-01-05T03:20:47.821269', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.004625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.032, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 749.0845489501953}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:20:48,220 INFO HandlerThread:2035853 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:20:48,220 INFO HandlerThread:2035853 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:20:48,220 DEBUG HandlerThread:2035853 [system_info.py:_save_code():44] Saving code +2025-01-04 22:20:48,222 DEBUG HandlerThread:2035853 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:20:48,222 DEBUG HandlerThread:2035853 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:20:48,284 DEBUG HandlerThread:2035853 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:20:48,284 INFO HandlerThread:2035853 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:20:48,287 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:20:48,287 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:20:48,288 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:20:48,288 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:20:48,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:20:48,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:20:48,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:20:48,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:20:48,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:48,520 INFO HandlerThread:2035853 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/tb" + save: true +} +control { + relay_id: "cf4338593ed94baa81a5415d41e7825f" +} +_info { + stream_id: "pzxh5op7" +} + +2025-01-04 22:20:48,521 DEBUG HandlerThread:2035853 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:20:48,535 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/vdc39fv3-wandb-metadata.json +2025-01-04 22:20:48,566 INFO wandb-upload_1:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/zya7rh51-code/FISH/eval_robot.py +2025-01-04 22:20:48,594 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/dskr9ok0-diff.patch +2025-01-04 22:20:48,677 DEBUG SenderThread:2035853 [sender.py:send():379] send: telemetry +2025-01-04 22:20:48,678 DEBUG SenderThread:2035853 [sender.py:send():379] send: tbrecord +2025-01-04 22:20:48,679 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:20:48,679 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 with policy live +2025-01-04 22:20:48,894 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/85jw3xng-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:20:49,107 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/diff.patch +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH/eval_robot.py +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-metadata.json +2025-01-04 22:20:49,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:20:49,109 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:49,109 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH +2025-01-04 22:20:49,109 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code +2025-01-04 22:20:49,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:50,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:51,105 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:51,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:52,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:53,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:53,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:53,704 DEBUG SenderThread:2035853 [sender.py:send():379] send: config +2025-01-04 22:20:54,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:55,106 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:55,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:56,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:57,106 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:57,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:58,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:58,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:59,107 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:20:59,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:00,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:01,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:01,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:02,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:03,108 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:03,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:03,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:03,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:03,632 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:04,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:05,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:06,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:07,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:08,521 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:09,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:09,521 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:10,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:11,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:12,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:13,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:14,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:15,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:15,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:16,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:17,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:18,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:18,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:18,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:19,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:20,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:20,925 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:21,113 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml +2025-01-04 22:21:21,113 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:21,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:22,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:23,114 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:23,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:24,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:25,114 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:25,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:26,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:26,964 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:27,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:28,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:29,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:30,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:31,116 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:31,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:32,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:32,861 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:33,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:33,518 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:33,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:34,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:35,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:36,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:37,118 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:37,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:38,228 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:38,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:39,118 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:21:39,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:40,526 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:41,526 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:42,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:43,521 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:43,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:44,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:45,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:46,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:47,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:48,204 DEBUG SystemMonitor:2035853 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:21:48,204 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:21:48,518 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:21:48,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:21:48,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:48,589 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:49,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:50,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:51,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:52,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:53,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:54,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:21:54,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:55,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:56,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:57,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:58,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:59,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:21:59,814 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:00,032 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:22:00,032 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 with policy now +2025-01-04 22:22:00,033 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:22:00,034 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:22:00,034 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:22:00,034 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media +2025-01-04 22:22:00,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos +2025-01-04 22:22:00,261 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/vhdbrt9c-media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:22:00,522 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/0xwpfcvj-media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:22:00,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:01,124 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:01,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:02,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:03,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:03,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:03,531 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:04,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:05,125 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:05,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:05,667 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:06,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:07,126 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:07,529 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:08,127 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:22:08,362 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/bsubc4ah-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:22:08,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:09,127 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:09,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:10,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:11,041 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:11,127 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:11,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:12,531 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:13,128 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:13,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:14,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:15,530 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:16,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:16,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:17,533 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:18,208 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:22:18,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:18,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:18,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:19,130 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:19,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:20,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:21,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:21,617 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:22,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:23,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:24,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:25,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:26,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:27,524 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:27,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:28,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:29,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:30,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:31,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:32,526 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:32,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:33,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:33,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:33,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:34,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:35,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:36,535 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:37,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:37,667 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:38,535 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:39,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:40,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:41,538 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:42,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:43,048 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:43,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:44,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:45,138 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:45,536 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:46,541 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:47,502 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:22:47,503 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 with policy now +2025-01-04 22:22:47,503 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:22:47,504 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:22:47,504 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:22:47,509 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:22:47,537 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:47,746 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/81gezt0y-media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:22:48,139 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:22:48,139 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:22:48,140 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:22:48,205 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:22:48,206 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:48,408 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/f35c584z-media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:22:48,535 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:22:48,535 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:22:48,577 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:49,139 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:49,544 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:50,541 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:51,543 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:52,546 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:53,141 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:53,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:53,548 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:54,550 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:55,142 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:22:55,143 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:55,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:55,966 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:22:55,966 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:22:55,966 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:22:56,143 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:22:56,548 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:57,143 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:22:57,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:58,533 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:22:58,550 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:22:59,550 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:00,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:01,144 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:01,548 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:02,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:03,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:03,519 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:03,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:03,654 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:04,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:05,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:06,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:07,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:08,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:09,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:09,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:10,549 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:11,551 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:12,551 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:13,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:14,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:15,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:15,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:16,552 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:17,553 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:18,206 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:23:18,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:18,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:18,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:19,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:20,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:20,608 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:21,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:22,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:23,554 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:24,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:25,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:26,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:26,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:27,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:28,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:29,555 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:30,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:31,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:32,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:32,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:33,519 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:33,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:33,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:34,556 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:35,155 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:35,561 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:36,133 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:23:36,133 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 with policy now +2025-01-04 22:23:36,135 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:23:36,136 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:23:36,136 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:23:36,143 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:23:36,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:23:36,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:23:36,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:23:36,353 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/xh41ftsh-media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:23:36,560 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:36,607 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/et5ojzyg-media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:23:37,156 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:37,566 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:38,164 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:38,567 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:39,562 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:40,563 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:41,157 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:41,565 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:42,564 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:43,158 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:43,409 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:43,568 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:44,566 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:45,159 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:45,571 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:46,570 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:47,568 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:48,206 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:23:48,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:23:48,525 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:23:48,576 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:48,651 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:49,574 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:50,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:51,577 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:52,581 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:53,576 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:53,718 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:53,820 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:23:53,820 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:23:53,821 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:23:54,162 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:23:54,162 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:23:54,398 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/87tlgr8t-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:23:54,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:55,162 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:55,586 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:56,578 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:57,163 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:57,576 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:58,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:23:58,792 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:23:59,163 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:23:59,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:00,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:01,164 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:01,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:02,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:03,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:03,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:03,575 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:04,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:04,657 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:05,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:06,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:07,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:08,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:09,167 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:09,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:10,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:10,579 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:11,581 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:12,581 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:13,582 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:14,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:15,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:15,582 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:16,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:17,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:18,207 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:24:18,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:18,520 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:18,583 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:19,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:20,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:20,594 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:21,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:22,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:23,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:24,584 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:25,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:26,528 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:26,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:27,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:28,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:29,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:30,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:31,174 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:31,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:31,674 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:32,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:33,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:33,521 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:33,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:34,594 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:34,689 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:24:34,689 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 with policy now +2025-01-04 22:24:34,694 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:24:34,694 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:24:34,694 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:24:34,705 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:24:34,930 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/ab0s6r8z-media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:24:35,175 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:35,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:24:35,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:24:35,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:24:35,442 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/ixmz1iwy-media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:24:35,591 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:36,619 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:36,714 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:37,176 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:37,607 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:38,602 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:39,177 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:39,600 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:40,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:41,609 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:41,765 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:42,610 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:43,178 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:43,605 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:43,878 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:24:43,878 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:24:43,879 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:24:44,179 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:24:44,179 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:24:44,610 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:45,179 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:45,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:46,608 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:47,180 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:47,253 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:47,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:48,207 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:24:48,527 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:24:48,527 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:24:48,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:49,180 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:49,609 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:50,629 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:51,181 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:51,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:52,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:52,773 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:53,182 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:24:53,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:54,611 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:55,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:56,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:57,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:58,531 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:24:58,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:24:59,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:00,612 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:01,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:02,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:03,520 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:03,521 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:03,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:03,680 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:04,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:05,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:06,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:07,613 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:08,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:09,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:09,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:10,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:11,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:12,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:13,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:14,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:15,532 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:15,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:16,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:17,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:18,208 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:25:18,522 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:18,522 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:18,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:19,614 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:20,607 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:20,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:21,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:22,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:23,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:24,615 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:25,635 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:25,987 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:26,267 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:25:26,267 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 with policy now +2025-01-04 22:25:26,276 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:25:26,276 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:26,277 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:26,278 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:26,501 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/j2wtwy25-media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:25:26,616 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:27,193 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:27,193 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:25:27,194 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:25:27,194 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval +2025-01-04 22:25:27,473 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/c4uaa5n6-media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:25:27,629 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:28,628 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:29,631 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:30,627 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:31,195 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:31,385 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:31,634 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:32,635 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:33,195 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:33,525 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:33,525 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:33,636 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:34,639 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:35,636 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:36,585 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:36,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:37,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:38,197 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:25:38,430 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/zkxgnqhx-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:25:38,648 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:38,867 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:25:38,868 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:25:39,022 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:25:39,197 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table +2025-01-04 22:25:39,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval +2025-01-04 22:25:39,373 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/58nt0szu-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:25:39,397 INFO wandb-upload_0:2035853 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4jies8t4 +2025-01-04 22:25:39,500 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:25:39,510 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:25:39,637 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:39,809 INFO SenderThread:2035853 [sender.py:send_request_log_artifact():1525] logged artifact run-pzxh5op7-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5MDY3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MjczMQ==', 'latestArtifact': None}} +2025-01-04 22:25:39,809 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:25:39,809 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 22:25:39,809 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:39,809 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,810 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,810 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:39,810 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,811 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,811 DEBUG SenderThread:2035853 [sender.py:send():379] send: telemetry +2025-01-04 22:25:39,811 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,812 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,813 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,813 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,813 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,813 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,814 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,814 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,814 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,814 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,814 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,815 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,815 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,815 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,815 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,815 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,816 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,816 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,816 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,816 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,817 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:39,817 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:39,817 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:25:40,048 INFO wandb-upload_1:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/wrb7lgly-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:25:40,186 INFO wandb-upload_0:2035853 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpizmnphev +2025-01-04 22:25:40,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:25:40,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:25:40,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:25:40,199 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval +2025-01-04 22:25:40,536 INFO SenderThread:2035853 [sender.py:send_request_log_artifact():1525] logged artifact run-pzxh5op7-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5MDY3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MjczMg==', 'latestArtifact': None}} +2025-01-04 22:25:40,536 DEBUG SenderThread:2035853 [sender.py:send():379] send: files +2025-01-04 22:25:40,536 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json with policy now +2025-01-04 22:25:40,537 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:25:40,537 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:25:40,537 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:25:40,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:40,716 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/dxi9kibz-media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:25:40,945 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/ng8ayly3-media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:25:41,198 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:25:41,199 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:25:41,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:42,541 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:42,645 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:43,199 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml +2025-01-04 22:25:43,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:44,640 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:45,641 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:46,641 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:47,641 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:48,208 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:25:48,208 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:48,523 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:25:48,523 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:25:48,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:49,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:50,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:51,642 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:52,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:53,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:53,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:54,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:55,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:56,643 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:57,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:58,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:25:59,534 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:25:59,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:00,644 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:00,833 DEBUG SenderThread:2035853 [sender.py:send():379] send: exit +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_exit():588] handling runtime: 312 +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_exit():594] send defer +2025-01-04 22:26:00,834 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,834 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:26:00,834 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:26:00,834 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:26:00,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,835 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:26:00,835 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,835 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:26:00,835 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:26:00,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,835 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:26:00,835 INFO HandlerThread:2035853 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:26:00,835 DEBUG SystemMonitor:2035853 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:26:00,835 DEBUG SystemMonitor:2035853 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:26:00,836 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:26:00,836 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:26:00,886 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:26:00,887 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:26:00,887 INFO HandlerThread:2035853 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:26:00,887 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,887 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:26:00,887 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:26:00,887 DEBUG SenderThread:2035853 [sender.py:send():379] send: stats +2025-01-04 22:26:00,887 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,888 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:26:00,888 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:00,888 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:26:00,888 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:26:00,888 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:00,888 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:26:01,206 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:26:03,207 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:26:06,950 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:06,950 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:06,950 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:26:06,951 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,951 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:26:06,951 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:26:06,952 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:06,952 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:06,952 DEBUG SenderThread:2035853 [sender.py:send():379] send: history +2025-01-04 22:26:06,952 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:26:06,952 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:26:06,953 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:26:06,953 DEBUG SenderThread:2035853 [sender.py:send():379] send: summary +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:26:06,954 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:26:06,954 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:06,954 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:26:06,954 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:26:06,954 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:26:06,954 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:06,954 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:06,954 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:26:06,955 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:06,955 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:26:07,041 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:26:07,041 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:07,041 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:26:07,041 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:07,041 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:26:07,041 INFO SenderThread:2035853 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:26:07,044 INFO SenderThread:2035853 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:26:07,046 INFO SenderThread:2035853 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:26:07,056 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:26:07,056 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:07,057 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:26:07,057 DEBUG SenderThread:2035853 [sender.py:send():379] send: artifact +2025-01-04 22:26:07,209 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:26:07,209 INFO Thread-12 :2035853 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:26:07,253 INFO SenderThread:2035853 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 22:26:07,253 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:07,253 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:26:07,253 INFO SenderThread:2035853 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:26:07,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml config.yaml +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json wandb-summary.json +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt requirements.txt +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:26:08,210 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log output.log +2025-01-04 22:26:08,214 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/diff.patch diff.patch +2025-01-04 22:26:08,214 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 media/videos/eval/1_eval_1_289145cdda89197ca8dd.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 media/videos/eval/3_eval_3_b257116aa31f0e2642be.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 media/videos/eval/4_eval_4_f785ef3290c03ae127b8.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 media/videos/eval/2_eval_2_f7358c10e28477967d40.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 media/videos/eval/0_eval_0_edb61c24cfb70619857b.mp4 +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json media/table/eval/total_success_rate_ci_6_82442c864ae09c6d7e48.table.json +2025-01-04 22:26:08,220 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:26:08,220 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,220 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,220 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:26:08,221 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,221 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:26:08,221 INFO SenderThread:2035853 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:26:08,397 INFO wandb-upload_2:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/config.yaml +2025-01-04 22:26:08,408 INFO wandb-upload_1:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/wandb-summary.json +2025-01-04 22:26:08,446 INFO wandb-upload_0:2035853 [upload_job.py:push():130] Uploaded file /tmp/tmp72wuxbovwandb/h5c5bxk7-events.out.tfevents.1736047248.leonmkim-ROG-Strix-G15CS-G15CS.2035658.0 +2025-01-04 22:26:08,454 INFO wandb-upload_3:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/requirements.txt +2025-01-04 22:26:08,490 INFO wandb-upload_4:2035853 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/files/output.log +2025-01-04 22:26:08,690 INFO Thread-11 (_thread_body):2035853 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:26:08,690 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,691 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:26:08,691 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,691 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:26:08,691 INFO SenderThread:2035853 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:26:08,692 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:26:08,692 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,692 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:26:08,692 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,692 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:26:08,693 INFO SenderThread:2035853 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:26:08,835 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,874 INFO SenderThread:2035853 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:26:08,874 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:26:08,874 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,875 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,875 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:26:08,875 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,875 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:26:08,875 INFO SenderThread:2035853 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:26:08,875 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:26:08,875 INFO HandlerThread:2035853 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:26:08,875 DEBUG SenderThread:2035853 [sender.py:send():379] send: final +2025-01-04 22:26:08,876 DEBUG SenderThread:2035853 [sender.py:send():379] send: footer +2025-01-04 22:26:08,876 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: defer +2025-01-04 22:26:08,876 INFO SenderThread:2035853 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:26:08,876 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,876 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,876 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:26:08,877 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:26:08,877 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:26:08,877 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:26:08,879 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:26:08,879 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:08,933 DEBUG SenderThread:2035853 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:26:08,934 INFO MainThread:2035853 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:26:08,934 INFO MainThread:2035853 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:26:08,934 INFO MainThread:2035853 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:26:08,935 DEBUG HandlerThread:2035853 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:26:08,935 INFO HandlerThread:2035853 [handler.py:finish():882] shutting down handler +2025-01-04 22:26:09,877 INFO WriterThread:2035853 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/run-pzxh5op7.wandb +2025-01-04 22:26:09,934 INFO SenderThread:2035853 [sender.py:finish():1615] shutting down sender +2025-01-04 22:26:09,934 INFO SenderThread:2035853 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:26:09,934 INFO SenderThread:2035853 [file_pusher.py:join():175] waiting for file pusher diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug.log b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0ebde1df55d07fbb8fd2f7052c285bb5cf70ff2d --- /dev/null +++ b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Configure stats pid to 2035658 +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/settings +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:20:47,827 INFO MainThread:2035658 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug.log +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/logs/debug-internal.log +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7'} +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():619] starting backend +2025-01-04 22:20:47,828 INFO MainThread:2035658 [wandb_init.py:init():623] setting up manager +2025-01-04 22:20:47,831 INFO MainThread:2035658 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:20:47,833 INFO MainThread:2035658 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:20:47,846 INFO MainThread:2035658 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:20:47,852 INFO MainThread:2035658 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:20:48,120 INFO MainThread:2035658 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:20:48,191 INFO MainThread:2035658 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:20:48,191 INFO MainThread:2035658 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:20:48,518 INFO MainThread:2035658 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:20:48,518 INFO MainThread:2035658 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:20:48,518 INFO MainThread:2035658 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:20:48,519 INFO MainThread:2035658 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:20:48,520 INFO MainThread:2035658 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:20:48,520 INFO MainThread:2035658 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/pzxh5op7/tb, True +2025-01-04 22:20:53,704 INFO MainThread:2035658 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:26:09,935 WARNING MsgRouterThr:2035658 [router.py:message_loop():77] message_loop has been closed diff --git a/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/run-pzxh5op7.wandb b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/run-pzxh5op7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7a78a4b75d8beeb07027091111138cdd5e63d3a3 Binary files /dev/null and b/pzxh5op7/wandb/run-20250104_222047-pzxh5op7/run-pzxh5op7.wandb differ diff --git a/snapshot_11749.pt b/snapshot_11749.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d06a7789aafe283c2ba8848f2797675c81ebd97 --- /dev/null +++ b/snapshot_11749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76a75fb2ac201a564d54518f9f5914d1c959961c9ef0468caaa55bd26ab11b2a +size 910446022 diff --git a/snapshot_11999.pt b/snapshot_11999.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8af601cc8f5f5d52bd6b58e02d0cd8cd9308c2f --- /dev/null +++ b/snapshot_11999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58dfcc225a4e34c9220a876941e9c1a38846846a1a9c03af2e1050db63ccf16a +size 910446022 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..91502222bdfcb1b7df8b9eb0b4618e0807be3307 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59d407eefe76863e23026c5f0e7a4595998fa5d29b80ad2fc460454ccdad82ae +size 910446022 diff --git a/snapshot_12079.pt b/snapshot_12079.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fbcd7809d971bf4d3d63e4feeec7c3305e28af9 --- /dev/null +++ b/snapshot_12079.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89cd3433f1e9b5fd5693c5c1e5a4fe82af081faef10485fbda5fe63f482d93e3 +size 910446022 diff --git a/snapshot_13749.pt b/snapshot_13749.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd48b58be8846baeca47b17fa27bb05b57072c60 --- /dev/null +++ b/snapshot_13749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fdb4640c77a3008074d7f288300a42cf4747092bbf5dbc74d33fadc24e31d4a +size 910446022 diff --git a/snapshot_4567.pt b/snapshot_4567.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ae0f224a766bae268390693265db01b06c941df --- /dev/null +++ b/snapshot_4567.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8123fafca81e1b4be6193df02555c664df321ee5e8a999bd921123a08613f2a4 +size 910446022 diff --git a/uymqqs40/.hydra/config.yaml b/uymqqs40/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/uymqqs40/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/uymqqs40/.hydra/hydra.yaml b/uymqqs40/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ac87a0c6c972a64d789d52fd239b8b445f82aa7 --- /dev/null +++ b/uymqqs40/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/uymqqs40/.hydra/overrides.yaml b/uymqqs40/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/uymqqs40/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/uymqqs40/episode_rosbags/aligned_depth_to_color_K.npy b/uymqqs40/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/uymqqs40/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/uymqqs40/episode_rosbags/cam_tf_world.npy b/uymqqs40/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/uymqqs40/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/uymqqs40/episode_rosbags/color_K.npy b/uymqqs40/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/uymqqs40/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/uymqqs40/episode_rosbags/depth_K.npy b/uymqqs40/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/uymqqs40/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/uymqqs40/episode_rosbags/episode_0_2025-01-04-22-26-57.bag b/uymqqs40/episode_rosbags/episode_0_2025-01-04-22-26-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..af33e20efcf97b620d7738740df6d77f816ee924 --- /dev/null +++ b/uymqqs40/episode_rosbags/episode_0_2025-01-04-22-26-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5af4f2d091538ef4775e73f557dcfbbde621a0ddb1236cb23d5e355b67f1c03 +size 1343238760 diff --git a/uymqqs40/eval_robot.log b/uymqqs40/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..e86905fb42119304c3066023ff97bf0b96347706 --- /dev/null +++ b/uymqqs40/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:26:29,758][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:26:29,760][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/uymqqs40/eval_video/0_eval.mp4 b/uymqqs40/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b83b5087d78f6a995682c1f7daeebf8877f2a982 --- /dev/null +++ b/uymqqs40/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13ad8d07b0c454797c1691bc4b4c12c00d8b861b9df9bd61d6b88e2fc2f8c1a +size 844077 diff --git a/uymqqs40/eval_video/1_eval.mp4 b/uymqqs40/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/uymqqs40/eval_video/1_eval.mp4 differ diff --git a/uymqqs40/tb/events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 b/uymqqs40/tb/events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 new file mode 100644 index 0000000000000000000000000000000000000000..57d887c48aa9e468f927c3bd0a75f440234185e6 --- /dev/null +++ b/uymqqs40/tb/events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:512d801a126833b6659b425fba47759dacd5c157e23f0549954b7fff19996f59 +size 88 diff --git a/uymqqs40/wandb/debug-internal.log b/uymqqs40/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..01e0d659dcccaeea0063a429efd73034c5aad424 --- /dev/null +++ b/uymqqs40/wandb/debug-internal.log @@ -0,0 +1,378 @@ +2025-01-04 22:26:35,546 INFO StreamThr :2036944 [internal.py:wandb_internal():85] W&B internal server running at pid: 2036944, started at: 2025-01-04 22:26:35.546392 +2025-01-04 22:26:35,547 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:26:35,559 INFO WriterThread:2036944 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/run-uymqqs40.wandb +2025-01-04 22:26:35,559 DEBUG SenderThread:2036944 [sender.py:send():379] send: header +2025-01-04 22:26:35,569 DEBUG SenderThread:2036944 [sender.py:send():379] send: run +2025-01-04 22:26:35,830 INFO SenderThread:2036944 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files +2025-01-04 22:26:35,830 INFO SenderThread:2036944 [sender.py:_start_run_threads():1188] run started: uymqqs40 with start time 1736047595.547723 +2025-01-04 22:26:35,846 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:26:35,846 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:26:35,925 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:26:35,929 DEBUG HandlerThread:2036944 [system_info.py:__init__():26] System info init +2025-01-04 22:26:35,929 DEBUG HandlerThread:2036944 [system_info.py:__init__():41] System info init done +2025-01-04 22:26:35,929 INFO HandlerThread:2036944 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:26:35,930 INFO SystemMonitor:2036944 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:26:35,930 INFO HandlerThread:2036944 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:26:35,930 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:26:35,931 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:26:35,932 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:26:35,933 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:26:35,933 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:26:35,943 DEBUG HandlerThread:2036944 [system_info.py:probe():152] Probing system +2025-01-04 22:26:35,945 DEBUG HandlerThread:2036944 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:26:35,949 DEBUG HandlerThread:2036944 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:26:35,949 DEBUG HandlerThread:2036944 [system_info.py:probe():200] Probing system done +2025-01-04 22:26:35,950 DEBUG HandlerThread:2036944 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:26:35.943916', 'startedAt': '2025-01-05T03:26:35.532415', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.9911250000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.966, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 761.2699356079102}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:26:35,950 INFO HandlerThread:2036944 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:26:35,950 INFO HandlerThread:2036944 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:26:35,950 DEBUG HandlerThread:2036944 [system_info.py:_save_code():44] Saving code +2025-01-04 22:26:35,952 DEBUG HandlerThread:2036944 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:26:35,952 DEBUG HandlerThread:2036944 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:26:36,014 DEBUG HandlerThread:2036944 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:26:36,014 INFO HandlerThread:2036944 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:26:36,017 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:26:36,017 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:26:36,018 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:26:36,018 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:26:36,251 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:26:36,251 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:26:36,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:26:36,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:26:36,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:36,254 INFO HandlerThread:2036944 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/tb" + save: true +} +control { + relay_id: "64aa79092c7046fea4f26362aebb0073" +} +_info { + stream_id: "uymqqs40" +} + +2025-01-04 22:26:36,254 DEBUG HandlerThread:2036944 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:26:36,260 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/h6tu1vcf-code/FISH/eval_robot.py +2025-01-04 22:26:36,279 INFO wandb-upload_2:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/pk1rr53r-diff.patch +2025-01-04 22:26:36,295 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/16li7rd5-wandb-metadata.json +2025-01-04 22:26:36,344 DEBUG SenderThread:2036944 [sender.py:send():379] send: telemetry +2025-01-04 22:26:36,345 DEBUG SenderThread:2036944 [sender.py:send():379] send: tbrecord +2025-01-04 22:26:36,345 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:26:36,345 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 with policy live +2025-01-04 22:26:36,528 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/4gdqox9q-events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH/eval_robot.py +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-metadata.json +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/diff.patch +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code +2025-01-04 22:26:37,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:38,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:38,831 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:39,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:40,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:41,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:41,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:41,396 DEBUG SenderThread:2036944 [sender.py:send():379] send: config +2025-01-04 22:26:42,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:42,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:43,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:44,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:44,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:45,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:46,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:46,834 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:47,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:47,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:48,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:49,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:50,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:50,835 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:51,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:26:51,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:26:51,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:52,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:52,357 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:53,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:54,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:54,836 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:55,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:56,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:56,836 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:57,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:57,917 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:58,259 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:59,261 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:00,261 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:00,838 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:01,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:02,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:03,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:03,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:04,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:05,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:06,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:06,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:06,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:07,264 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:08,264 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:08,403 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:08,840 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml +2025-01-04 22:27:09,264 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:10,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:11,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:12,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:13,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:14,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:14,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:15,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:16,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:17,068 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:27:17,068 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 with policy now +2025-01-04 22:27:17,068 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:27:17,069 DEBUG SenderThread:2036944 [sender.py:send():379] send: history +2025-01-04 22:27:17,069 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:27:17,070 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:27:17,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:17,300 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/f9mf61gb-media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:27:17,842 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media +2025-01-04 22:27:18,105 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/jr8v8bql-media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:27:18,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:18,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:19,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:20,082 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:20,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:21,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:21,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:21,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:22,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:22,844 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:23,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:24,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:24,845 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:25,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:26,046 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:26,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:27,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:28,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:29,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:30,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:31,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:31,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:32,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:33,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:34,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:35,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:35,933 DEBUG SystemMonitor:2036944 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:27:35,934 DEBUG SenderThread:2036944 [sender.py:send():379] send: stats +2025-01-04 22:27:36,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:36,253 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:36,297 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:36,345 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:37,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:38,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:39,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:40,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:41,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:42,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:42,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:43,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:44,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:45,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:46,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:47,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:47,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:48,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:49,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:50,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:51,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:51,253 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:51,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:52,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:52,399 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:53,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:54,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:55,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:56,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:57,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:58,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:58,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:59,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:00,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:01,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:02,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:03,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:03,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:04,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:05,272 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:05,934 DEBUG SenderThread:2036944 [sender.py:send():379] send: stats +2025-01-04 22:28:06,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:28:06,253 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:28:06,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:07,272 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:08,278 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:08,357 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:08,452 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:28:08,452 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-04 22:28:08,452 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:28:08,452 DEBUG SenderThread:2036944 [sender.py:send():379] send: history +2025-01-04 22:28:08,452 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:28:08,453 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:28:08,454 DEBUG SenderThread:2036944 [sender.py:send():379] send: telemetry +2025-01-04 22:28:08,457 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:28:08,457 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:28:08,524 DEBUG SenderThread:2036944 [sender.py:send():379] send: exit +2025-01-04 22:28:08,524 INFO SenderThread:2036944 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:28:08,524 INFO SenderThread:2036944 [sender.py:send_exit():588] handling runtime: 92 +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:send_exit():594] send defer +2025-01-04 22:28:08,525 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,525 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:28:08,525 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:28:08,526 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,526 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:28:08,526 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,526 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:28:08,526 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:28:08,526 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,526 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:28:08,526 INFO HandlerThread:2036944 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:28:08,526 DEBUG SystemMonitor:2036944 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:28:08,526 DEBUG SystemMonitor:2036944 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:28:08,527 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:28:08,527 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:28:08,577 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:28:08,577 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:28:08,577 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:28:08,578 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,578 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:28:08,578 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:28:08,578 DEBUG SenderThread:2036944 [sender.py:send():379] send: stats +2025-01-04 22:28:08,578 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,578 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:28:08,579 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,579 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:28:08,579 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:28:08,579 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,579 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:28:08,641 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/vy6twug6-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval +2025-01-04 22:28:09,096 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/mijpqbwa-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:10,860 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:13,861 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml +2025-01-04 22:28:15,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:15,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:15,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:28:15,266 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,266 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:28:15,266 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:28:15,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:15,266 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:28:15,266 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:15,266 DEBUG SenderThread:2036944 [sender.py:send():379] send: summary +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:28:15,267 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:28:15,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:15,267 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:28:15,267 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:28:15,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:15,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:15,268 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:28:15,268 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,268 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:28:15,565 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:15,861 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:28:16,451 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:28:16,451 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:16,451 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:16,452 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:28:16,452 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:16,452 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:28:16,452 INFO SenderThread:2036944 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:28:16,454 INFO SenderThread:2036944 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:28:16,468 INFO SenderThread:2036944 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:28:16,482 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:28:16,483 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:16,483 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:28:16,483 DEBUG SenderThread:2036944 [sender.py:send():379] send: artifact +2025-01-04 22:28:16,569 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:16,862 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:16,979 INFO wandb-upload_2:2036944 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6169m_y9 +2025-01-04 22:28:17,090 INFO wandb-upload_0:2036944 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_8l7m_n1 +2025-01-04 22:28:17,157 INFO wandb-upload_1:2036944 [upload_job.py:push():88] Uploaded file /tmp/tmphevsx_qz/wandb-job.json +2025-01-04 22:28:17,759 INFO SenderThread:2036944 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 22:28:17,759 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:17,759 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:28:17,759 INFO SenderThread:2036944 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:28:17,862 INFO SenderThread:2036944 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files +2025-01-04 22:28:17,862 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml config.yaml +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json wandb-summary.json +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt requirements.txt +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log output.log +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/diff.patch diff.patch +2025-01-04 22:28:17,866 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:28:17,869 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:28:17,869 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:17,869 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:28:17,869 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:17,869 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:17,869 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:28:17,870 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:17,870 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:28:17,870 INFO SenderThread:2036944 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:28:18,037 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt +2025-01-04 22:28:18,038 INFO wandb-upload_3:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:28:18,044 INFO wandb-upload_2:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml +2025-01-04 22:28:18,128 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:18,328 INFO Thread-11 (_thread_body):2036944 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:28:18,328 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,328 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:28:18,328 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:28:18,329 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,329 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:28:18,329 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:28:18,330 INFO SenderThread:2036944 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:28:18,400 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,400 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:28:18,400 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:28:18,401 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,401 DEBUG SenderThread:2036944 [sender.py:send():379] send: final +2025-01-04 22:28:18,401 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:28:18,401 DEBUG SenderThread:2036944 [sender.py:send():379] send: footer +2025-01-04 22:28:18,401 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,401 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:28:18,421 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:18,421 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:18,437 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:18,443 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:28:18,446 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:28:18,446 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:28:18,446 INFO HandlerThread:2036944 [handler.py:finish():882] shutting down handler +2025-01-04 22:28:19,421 INFO WriterThread:2036944 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/run-uymqqs40.wandb +2025-01-04 22:28:19,422 INFO SenderThread:2036944 [sender.py:finish():1615] shutting down sender +2025-01-04 22:28:19,422 INFO SenderThread:2036944 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:28:19,422 INFO SenderThread:2036944 [file_pusher.py:join():175] waiting for file pusher diff --git a/uymqqs40/wandb/debug.log b/uymqqs40/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..91782d5a5e9f4c704c4b0f4982ac276a4ed08bfa --- /dev/null +++ b/uymqqs40/wandb/debug.log @@ -0,0 +1,37 @@ +2025-01-04 22:26:35,539 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Configure stats pid to 2036770 +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/settings +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug.log +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug-internal.log +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:26:35,541 INFO MainThread:2036770 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40'} +2025-01-04 22:26:35,541 INFO MainThread:2036770 [wandb_init.py:init():619] starting backend +2025-01-04 22:26:35,541 INFO MainThread:2036770 [wandb_init.py:init():623] setting up manager +2025-01-04 22:26:35,545 INFO MainThread:2036770 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:26:35,547 INFO MainThread:2036770 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:26:35,559 INFO MainThread:2036770 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:26:35,566 INFO MainThread:2036770 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:26:35,846 INFO MainThread:2036770 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:26:35,919 INFO MainThread:2036770 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:26:35,920 INFO MainThread:2036770 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:26:36,253 INFO MainThread:2036770 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:26:36,254 INFO MainThread:2036770 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/tb, True +2025-01-04 22:26:41,395 INFO MainThread:2036770 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:28:08,452 INFO MainThread:2036770 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/uymqqs40 +2025-01-04 22:28:08,454 INFO MainThread:2036770 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-04 22:28:08,454 INFO MainThread:2036770 [wandb_run.py:_restore():2352] restore +2025-01-04 22:28:08,455 INFO MainThread:2036770 [wandb_run.py:_restore():2358] restore done +2025-01-04 22:28:19,449 INFO MainThread:2036770 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:28:19,449 INFO MainThread:2036770 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:28:19,512 INFO MainThread:2036770 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH/eval_robot.py b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50ae9eb03b0014c9ec7df08fd8a7f691d1fb9d93 --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736047595 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 2 + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/diff.patch b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b83b5087d78f6a995682c1f7daeebf8877f2a982 --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13ad8d07b0c454797c1691bc4b4c12c00d8b861b9df9bd61d6b88e2fc2f8c1a +size 844077 diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 differ diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..cfb38b8c67fd4f4cca1859a1be9d4fd559d8793d --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log @@ -0,0 +1,60 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736047601.396258]: resetting environment +[INFO] [1736047601.400555]: cleared current plan +[INFO] [1736047601.401849]: moving to home +[INFO] [1736047602.404753]: reached home +[INFO] [1736047602.405976]: reset action history +[INFO] [1736047605.118922]: environment reset +Starting episode 0 +[INFO] [1736047605.123052]: resetting environment +[INFO] [1736047605.127915]: cleared current plan +[INFO] [1736047605.128896]: moving to home +[INFO] [1736047606.131174]: reached home +[INFO] [1736047606.132513]: reset action history +[INFO] [1736047608.842630]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736047614.147741]: resetting environment +[INFO] [1736047614.154547]: cleared current plan +[INFO] [1736047614.155394]: moving to home +[INFO] [1736047615.157263]: reached home +[INFO] [1736047615.158881]: reset action history +[INFO] [1736047617.916546]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/episode_rosbags/episode_0_2025-01-04-22-26-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2036770.log +started roslaunch server http://158.130.50.37:41151/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2037061] +process[print_text-2]: started with pid [2037085] +process[rosbag_record-3]: started with pid [2037086] +started bagging! +preempting episode +[INFO] [1736047636.085814]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736047636.839247]: Stored episode 1. +[INFO] [1736047636.839486]: Saving video... +[INFO] [1736047637.050516]: Video saved! +Starting episode 1 +[INFO] [1736047637.068918]: resetting environment +[INFO] [1736047637.081353]: cleared current plan +[INFO] [1736047637.081982]: moving to home +[INFO] [1736047640.284253]: reached home +[INFO] [1736047640.285214]: reset action history +[INFO] [1736047642.999161]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1736047688.358728]: Saving video... +[INFO] [1736047688.360194]: Video saved! \ No newline at end of file diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-metadata.json b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..13a7905d4ad4b87a5f3518194575f3fcedc788df --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:26:35.943916", + "startedAt": "2025-01-05T03:26:35.532415", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.9911250000005, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.966, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 761.2699356079102 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..301ec0a41be26f00fc7ac6f563199727629933cb --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "d13ad8d07b0c454797c1691bc4b4c12c00d8b861b9df9bd61d6b88e2fc2f8c1a", "size": 844077, "path": "media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4"}, "global_step": 68, "_timestamp": 1736047688.4520552, "_runtime": 92.9043321609497, "_step": 1, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 92}} \ No newline at end of file diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug-internal.log b/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..01e0d659dcccaeea0063a429efd73034c5aad424 --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug-internal.log @@ -0,0 +1,378 @@ +2025-01-04 22:26:35,546 INFO StreamThr :2036944 [internal.py:wandb_internal():85] W&B internal server running at pid: 2036944, started at: 2025-01-04 22:26:35.546392 +2025-01-04 22:26:35,547 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:26:35,559 INFO WriterThread:2036944 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/run-uymqqs40.wandb +2025-01-04 22:26:35,559 DEBUG SenderThread:2036944 [sender.py:send():379] send: header +2025-01-04 22:26:35,569 DEBUG SenderThread:2036944 [sender.py:send():379] send: run +2025-01-04 22:26:35,830 INFO SenderThread:2036944 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files +2025-01-04 22:26:35,830 INFO SenderThread:2036944 [sender.py:_start_run_threads():1188] run started: uymqqs40 with start time 1736047595.547723 +2025-01-04 22:26:35,846 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:26:35,846 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:26:35,925 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:26:35,929 DEBUG HandlerThread:2036944 [system_info.py:__init__():26] System info init +2025-01-04 22:26:35,929 DEBUG HandlerThread:2036944 [system_info.py:__init__():41] System info init done +2025-01-04 22:26:35,929 INFO HandlerThread:2036944 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:26:35,930 INFO SystemMonitor:2036944 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:26:35,930 INFO HandlerThread:2036944 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:26:35,930 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:26:35,931 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:26:35,932 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:26:35,933 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:26:35,933 INFO SystemMonitor:2036944 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:26:35,943 DEBUG HandlerThread:2036944 [system_info.py:probe():152] Probing system +2025-01-04 22:26:35,945 DEBUG HandlerThread:2036944 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:26:35,949 DEBUG HandlerThread:2036944 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:26:35,949 DEBUG HandlerThread:2036944 [system_info.py:probe():200] Probing system done +2025-01-04 22:26:35,950 DEBUG HandlerThread:2036944 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:26:35.943916', 'startedAt': '2025-01-05T03:26:35.532415', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.9911250000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.966, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 761.2699356079102}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:26:35,950 INFO HandlerThread:2036944 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:26:35,950 INFO HandlerThread:2036944 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:26:35,950 DEBUG HandlerThread:2036944 [system_info.py:_save_code():44] Saving code +2025-01-04 22:26:35,952 DEBUG HandlerThread:2036944 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:26:35,952 DEBUG HandlerThread:2036944 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:26:36,014 DEBUG HandlerThread:2036944 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:26:36,014 INFO HandlerThread:2036944 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:26:36,017 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:26:36,017 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:26:36,018 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:26:36,018 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:26:36,251 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:26:36,251 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:26:36,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:26:36,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:26:36,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:36,254 INFO HandlerThread:2036944 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/tb" + save: true +} +control { + relay_id: "64aa79092c7046fea4f26362aebb0073" +} +_info { + stream_id: "uymqqs40" +} + +2025-01-04 22:26:36,254 DEBUG HandlerThread:2036944 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:26:36,260 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/h6tu1vcf-code/FISH/eval_robot.py +2025-01-04 22:26:36,279 INFO wandb-upload_2:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/pk1rr53r-diff.patch +2025-01-04 22:26:36,295 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/16li7rd5-wandb-metadata.json +2025-01-04 22:26:36,344 DEBUG SenderThread:2036944 [sender.py:send():379] send: telemetry +2025-01-04 22:26:36,345 DEBUG SenderThread:2036944 [sender.py:send():379] send: tbrecord +2025-01-04 22:26:36,345 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:26:36,345 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 with policy live +2025-01-04 22:26:36,528 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/4gdqox9q-events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH/eval_robot.py +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 +2025-01-04 22:26:36,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-metadata.json +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/diff.patch +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH +2025-01-04 22:26:36,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code +2025-01-04 22:26:37,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:38,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:38,831 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:39,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:40,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:41,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:41,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:41,396 DEBUG SenderThread:2036944 [sender.py:send():379] send: config +2025-01-04 22:26:42,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:42,832 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:43,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:44,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:44,833 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:45,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:46,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:46,834 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:47,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:47,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:48,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:49,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:50,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:50,835 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:51,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:26:51,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:26:51,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:52,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:52,357 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:53,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:54,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:54,836 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:55,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:56,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:56,836 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:26:57,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:57,917 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:26:58,259 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:26:59,261 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:00,261 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:00,838 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:01,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:02,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:03,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:03,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:04,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:05,263 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:06,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:06,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:06,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:07,264 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:08,264 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:08,403 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:08,840 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml +2025-01-04 22:27:09,264 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:10,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:11,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:12,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:13,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:14,253 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:14,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:15,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:16,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:17,068 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:27:17,068 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 with policy now +2025-01-04 22:27:17,068 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:27:17,069 DEBUG SenderThread:2036944 [sender.py:send():379] send: history +2025-01-04 22:27:17,069 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:27:17,070 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:27:17,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:17,300 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/f9mf61gb-media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:27:17,842 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos +2025-01-04 22:27:17,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media +2025-01-04 22:27:18,105 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/jr8v8bql-media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:27:18,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:18,843 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:19,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:20,082 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:20,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:21,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:21,252 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:21,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:22,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:22,844 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:23,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:24,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:24,845 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:27:25,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:26,046 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:26,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:27,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:28,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:29,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:30,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:31,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:31,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:32,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:33,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:34,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:35,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:35,933 DEBUG SystemMonitor:2036944 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:27:35,934 DEBUG SenderThread:2036944 [sender.py:send():379] send: stats +2025-01-04 22:27:36,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:36,253 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:36,297 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:36,345 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:37,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:38,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:39,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:40,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:41,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:42,254 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:42,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:43,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:44,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:45,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:46,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:47,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:47,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:48,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:49,269 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:50,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:51,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:27:51,253 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:27:51,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:52,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:52,399 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:53,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:54,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:55,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:56,270 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:57,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:58,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:27:58,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:27:59,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:00,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:01,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:02,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:03,255 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:03,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:04,271 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:05,272 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:05,934 DEBUG SenderThread:2036944 [sender.py:send():379] send: stats +2025-01-04 22:28:06,252 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:28:06,253 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:28:06,293 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:07,272 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:08,278 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:08,357 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:08,452 DEBUG SenderThread:2036944 [sender.py:send():379] send: files +2025-01-04 22:28:08,452 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-04 22:28:08,452 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:28:08,452 DEBUG SenderThread:2036944 [sender.py:send():379] send: history +2025-01-04 22:28:08,452 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:28:08,453 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:28:08,454 DEBUG SenderThread:2036944 [sender.py:send():379] send: telemetry +2025-01-04 22:28:08,457 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:28:08,457 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:28:08,524 DEBUG SenderThread:2036944 [sender.py:send():379] send: exit +2025-01-04 22:28:08,524 INFO SenderThread:2036944 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:28:08,524 INFO SenderThread:2036944 [sender.py:send_exit():588] handling runtime: 92 +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:send_exit():594] send defer +2025-01-04 22:28:08,525 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,525 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:28:08,525 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:28:08,525 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:28:08,526 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,526 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:28:08,526 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,526 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:28:08,526 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:28:08,526 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,526 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:28:08,526 INFO HandlerThread:2036944 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:28:08,526 DEBUG SystemMonitor:2036944 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:28:08,526 DEBUG SystemMonitor:2036944 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:28:08,527 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:28:08,527 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:28:08,577 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:28:08,577 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:28:08,577 INFO HandlerThread:2036944 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:28:08,578 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,578 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:28:08,578 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:28:08,578 DEBUG SenderThread:2036944 [sender.py:send():379] send: stats +2025-01-04 22:28:08,578 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,578 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:28:08,579 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:08,579 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:28:08,579 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:28:08,579 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:08,579 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:28:08,641 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/vy6twug6-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:08,859 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval +2025-01-04 22:28:09,096 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /tmp/tmpkljj6cczwandb/mijpqbwa-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:10,860 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:13,861 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml +2025-01-04 22:28:15,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:15,265 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:15,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:28:15,266 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,266 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:28:15,266 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:28:15,266 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:15,266 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:28:15,266 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:15,266 DEBUG SenderThread:2036944 [sender.py:send():379] send: summary +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:28:15,267 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:28:15,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:15,267 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:28:15,267 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:28:15,267 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:28:15,267 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:15,268 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:15,268 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:28:15,268 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:15,268 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:28:15,565 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:15,861 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:28:16,451 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:28:16,451 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:16,451 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:16,452 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:28:16,452 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:16,452 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:28:16,452 INFO SenderThread:2036944 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:28:16,454 INFO SenderThread:2036944 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:28:16,468 INFO SenderThread:2036944 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:28:16,482 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:28:16,483 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:16,483 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:28:16,483 DEBUG SenderThread:2036944 [sender.py:send():379] send: artifact +2025-01-04 22:28:16,569 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:16,862 INFO Thread-12 :2036944 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:16,979 INFO wandb-upload_2:2036944 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6169m_y9 +2025-01-04 22:28:17,090 INFO wandb-upload_0:2036944 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_8l7m_n1 +2025-01-04 22:28:17,157 INFO wandb-upload_1:2036944 [upload_job.py:push():88] Uploaded file /tmp/tmphevsx_qz/wandb-job.json +2025-01-04 22:28:17,759 INFO SenderThread:2036944 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 22:28:17,759 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:17,759 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:28:17,759 INFO SenderThread:2036944 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:28:17,862 INFO SenderThread:2036944 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files +2025-01-04 22:28:17,862 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml config.yaml +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json wandb-summary.json +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 events.out.tfevents.1736047596.leonmkim-ROG-Strix-G15CS-G15CS.2036770.0 +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt requirements.txt +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log output.log +2025-01-04 22:28:17,863 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/diff.patch diff.patch +2025-01-04 22:28:17,866 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:28:17,869 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 media/videos/eval/0_eval_0_d13ad8d07b0c454797c1.mp4 +2025-01-04 22:28:17,869 INFO SenderThread:2036944 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2025-01-04 22:28:17,869 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:28:17,869 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:17,869 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:17,869 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:28:17,870 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:17,870 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:28:17,870 INFO SenderThread:2036944 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:28:18,037 INFO wandb-upload_0:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/requirements.txt +2025-01-04 22:28:18,038 INFO wandb-upload_3:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/wandb-summary.json +2025-01-04 22:28:18,044 INFO wandb-upload_2:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/config.yaml +2025-01-04 22:28:18,128 INFO wandb-upload_1:2036944 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/files/output.log +2025-01-04 22:28:18,328 INFO Thread-11 (_thread_body):2036944 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:28:18,328 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,328 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:28:18,328 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:28:18,329 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,329 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:28:18,329 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,329 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:28:18,330 INFO SenderThread:2036944 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:28:18,400 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,400 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:28:18,400 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:28:18,400 INFO SenderThread:2036944 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:28:18,401 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:28:18,401 DEBUG SenderThread:2036944 [sender.py:send():379] send: final +2025-01-04 22:28:18,401 INFO HandlerThread:2036944 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:28:18,401 DEBUG SenderThread:2036944 [sender.py:send():379] send: footer +2025-01-04 22:28:18,401 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: defer +2025-01-04 22:28:18,401 INFO SenderThread:2036944 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:28:18,421 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:28:18,421 DEBUG SenderThread:2036944 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:28:18,437 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:18,443 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:28:18,446 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:28:18,446 DEBUG HandlerThread:2036944 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:28:18,446 INFO HandlerThread:2036944 [handler.py:finish():882] shutting down handler +2025-01-04 22:28:19,421 INFO WriterThread:2036944 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/run-uymqqs40.wandb +2025-01-04 22:28:19,422 INFO SenderThread:2036944 [sender.py:finish():1615] shutting down sender +2025-01-04 22:28:19,422 INFO SenderThread:2036944 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:28:19,422 INFO SenderThread:2036944 [file_pusher.py:join():175] waiting for file pusher diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug.log b/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..91782d5a5e9f4c704c4b0f4982ac276a4ed08bfa --- /dev/null +++ b/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug.log @@ -0,0 +1,37 @@ +2025-01-04 22:26:35,539 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Configure stats pid to 2036770 +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/settings +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug.log +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/wandb/run-20250104_222635-uymqqs40/logs/debug-internal.log +2025-01-04 22:26:35,540 INFO MainThread:2036770 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:26:35,541 INFO MainThread:2036770 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40'} +2025-01-04 22:26:35,541 INFO MainThread:2036770 [wandb_init.py:init():619] starting backend +2025-01-04 22:26:35,541 INFO MainThread:2036770 [wandb_init.py:init():623] setting up manager +2025-01-04 22:26:35,545 INFO MainThread:2036770 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:26:35,547 INFO MainThread:2036770 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:26:35,559 INFO MainThread:2036770 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:26:35,566 INFO MainThread:2036770 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:26:35,846 INFO MainThread:2036770 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:26:35,919 INFO MainThread:2036770 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:26:35,920 INFO MainThread:2036770 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:26:36,252 INFO MainThread:2036770 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:26:36,253 INFO MainThread:2036770 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:26:36,254 INFO MainThread:2036770 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/uymqqs40/tb, True +2025-01-04 22:26:41,395 INFO MainThread:2036770 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:28:08,452 INFO MainThread:2036770 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/uymqqs40 +2025-01-04 22:28:08,454 INFO MainThread:2036770 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-04 22:28:08,454 INFO MainThread:2036770 [wandb_run.py:_restore():2352] restore +2025-01-04 22:28:08,455 INFO MainThread:2036770 [wandb_run.py:_restore():2358] restore done +2025-01-04 22:28:19,449 INFO MainThread:2036770 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:28:19,449 INFO MainThread:2036770 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:28:19,512 INFO MainThread:2036770 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/uymqqs40/wandb/run-20250104_222635-uymqqs40/run-uymqqs40.wandb b/uymqqs40/wandb/run-20250104_222635-uymqqs40/run-uymqqs40.wandb new file mode 100644 index 0000000000000000000000000000000000000000..174dec68d6e8401de24fff9d6757147044c4d0b2 Binary files /dev/null and b/uymqqs40/wandb/run-20250104_222635-uymqqs40/run-uymqqs40.wandb differ diff --git a/y2m69p6z/episode_rosbags/episode_0_2025-01-04-20-43-31.bag b/y2m69p6z/episode_rosbags/episode_0_2025-01-04-20-43-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..f32bc96bdb410f2efed4c3767f6ef38eff45af04 --- /dev/null +++ b/y2m69p6z/episode_rosbags/episode_0_2025-01-04-20-43-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d37e927c4975498d0bfc41cc3ec7e5b319d5248503b020f4d38321e5e79a7d +size 2606309043 diff --git a/y2m69p6z/episode_rosbags/episode_2_2025-01-04-20-45-26.bag b/y2m69p6z/episode_rosbags/episode_2_2025-01-04-20-45-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..d029fee543411aa72169053554b5af14887f1e0c --- /dev/null +++ b/y2m69p6z/episode_rosbags/episode_2_2025-01-04-20-45-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cd5ee95e8abfdfbcebd1b958854beb496e8f3104707c36ef113a8fbb75e9428 +size 2190906817 diff --git a/y2m69p6z/episode_rosbags/episode_4_2025-01-04-20-47-19.bag b/y2m69p6z/episode_rosbags/episode_4_2025-01-04-20-47-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..6af575bc516b903094f8819cc76c0aeb8f6e639a --- /dev/null +++ b/y2m69p6z/episode_rosbags/episode_4_2025-01-04-20-47-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9952600af87873aa6666d2ab29bc3d7fa2f3625013310639c73b2c7b594cc1ed +size 1634422885 diff --git a/y2m69p6z/eval_video/0_eval.mp4 b/y2m69p6z/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1607e6623b8d94938d04c4c004d34505152a6dea --- /dev/null +++ b/y2m69p6z/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4268b13f1b81246fc5ef8324c29a2330d3937259bbffbdcde004512ec893fb5c +size 1520919