diff --git a/.gitattributes b/.gitattributes index c2762adbf07f553b242ddbed308eec7aabe4b651..659783059b21d82677d9af36d81b459688b5698c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -78,3 +78,50 @@ ot5wit9y/episode_rosbags/episode_4_2025-01-04-21-12-56.bag filter=lfs diff=lfs m 6mzw1wsh/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text 6mzw1wsh/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text 6mzw1wsh/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/episode_rosbags/episode_1_2025-01-03-22-17-31.bag filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/episode_rosbags/episode_2_2025-01-03-22-18-35.bag filter=lfs diff=lfs merge=lfs -text +29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +29yeoadw/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/episode_rosbags/episode_3_2025-01-03-22-19-19.bag filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/episode_rosbags/episode_4_2025-01-03-22-20-02.bag filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/episode_rosbags/episode_0_2025-01-03-22-16-52.bag filter=lfs diff=lfs merge=lfs -text +29yeoadw/episode_rosbags/episode_4_2025-01-04-21-01-47.bag filter=lfs diff=lfs merge=lfs -text +29yeoadw/episode_rosbags/episode_3_2025-01-04-21-01-03.bag filter=lfs diff=lfs merge=lfs -text +29yeoadw/episode_rosbags/episode_2_2025-01-04-21-00-22.bag filter=lfs diff=lfs merge=lfs -text +cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +cxddxu7z/episode_rosbags/episode_2_2025-01-04-23-00-34.bag filter=lfs diff=lfs merge=lfs -text +cxddxu7z/episode_rosbags/episode_1_2025-01-04-22-59-32.bag filter=lfs diff=lfs merge=lfs -text +cxddxu7z/episode_rosbags/episode_0_2025-01-04-22-57-56.bag filter=lfs diff=lfs merge=lfs -text +l2vooc6k/episode_rosbags/episode_3_2025-01-04-21-06-04.bag filter=lfs diff=lfs merge=lfs -text +xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/episode_rosbags/episode_4_2025-01-04-21-06-53.bag filter=lfs diff=lfs merge=lfs -text +xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/episode_rosbags/episode_1_2025-01-04-21-04-42.bag filter=lfs diff=lfs merge=lfs -text +xt2ne5el/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/episode_rosbags/episode_2_2025-01-04-21-05-23.bag filter=lfs diff=lfs merge=lfs -text +cxddxu7z/episode_rosbags/episode_4_2025-01-04-23-02-06.bag filter=lfs diff=lfs merge=lfs -text +l2vooc6k/episode_rosbags/episode_0_2025-01-04-21-04-00.bag filter=lfs diff=lfs merge=lfs -text diff --git a/29yeoadw/.hydra/config.yaml b/29yeoadw/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/29yeoadw/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/29yeoadw/.hydra/hydra.yaml b/29yeoadw/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed8bcdf543b73180e11b47f57f5a902c20d11f85 --- /dev/null +++ b/29yeoadw/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/29yeoadw/.hydra/overrides.yaml b/29yeoadw/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/29yeoadw/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/29yeoadw/episode_rosbags/aligned_depth_to_color_K.npy b/29yeoadw/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/29yeoadw/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/29yeoadw/episode_rosbags/cam_tf_world.npy b/29yeoadw/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/29yeoadw/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/29yeoadw/episode_rosbags/color_K.npy b/29yeoadw/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/29yeoadw/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/29yeoadw/episode_rosbags/depth_K.npy b/29yeoadw/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/29yeoadw/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/29yeoadw/episode_rosbags/episode_2_2025-01-04-21-00-22.bag b/29yeoadw/episode_rosbags/episode_2_2025-01-04-21-00-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..02a82db22d1a946bce5f8bfa21f50f1e30ec6eb9 --- /dev/null +++ b/29yeoadw/episode_rosbags/episode_2_2025-01-04-21-00-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0810f2afa1e2f707601dcb54d0cba80c7c1f5c12306f4376f65bd0d315343d2 +size 1413894059 diff --git a/29yeoadw/episode_rosbags/episode_3_2025-01-04-21-01-03.bag b/29yeoadw/episode_rosbags/episode_3_2025-01-04-21-01-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..5552c56baadaa174eb35dbf3926c6ae1c21d30c6 --- /dev/null +++ b/29yeoadw/episode_rosbags/episode_3_2025-01-04-21-01-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10c0ab456a38a714fbffcb1257eec33e000b2a2eba4a0e41fb8cb243ac76350b +size 1451726131 diff --git a/29yeoadw/episode_rosbags/episode_4_2025-01-04-21-01-47.bag b/29yeoadw/episode_rosbags/episode_4_2025-01-04-21-01-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..6453644ff1d8f38edc73f8d2b0ac3392fd6ae94a --- /dev/null +++ b/29yeoadw/episode_rosbags/episode_4_2025-01-04-21-01-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:100707c3fc7d57f545f1ca5fab4cde5b76ab5ecb5748e2891cf78345b756c636 +size 1412282339 diff --git a/29yeoadw/eval_robot.log b/29yeoadw/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..aca7ef0cb9b64f59da622c72b4e20679a4ea6a61 --- /dev/null +++ b/29yeoadw/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 20:58:14,884][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:58:14,887][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/29yeoadw/eval_video/0_eval.mp4 b/29yeoadw/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48d9215418f2da65aa05163acad160d1401de1f3 --- /dev/null +++ b/29yeoadw/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:467f8a1696f4f1f555ef1602b3aa06d3d64c6bdfc8557e1daa693e7203a73150 +size 1159188 diff --git a/29yeoadw/eval_video/1_eval.mp4 b/29yeoadw/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..30849791339a96e6c3438c7db6808aeafe49b112 --- /dev/null +++ b/29yeoadw/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b99807ad0b8eaad92e7b7147d72177b57495022891ead586a4e3e37c39171b96 +size 1234058 diff --git a/29yeoadw/eval_video/2_eval.mp4 b/29yeoadw/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..532f58a228949647305684c7c8fe218b3f08dbb1 --- /dev/null +++ b/29yeoadw/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6008c3bedf6112092f657777e0f56a1ee6c9ab080a2d2e2c89bf64d227979595 +size 1166730 diff --git a/29yeoadw/eval_video/3_eval.mp4 b/29yeoadw/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d549e08b44426fd2a2d12f0209a0938a546f57bb --- /dev/null +++ b/29yeoadw/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f38e890841ba060d24fd1960b3b7f0da23b9be2a3a5464578984e1271c3566a +size 1213103 diff --git a/29yeoadw/eval_video/4_eval.mp4 b/29yeoadw/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4e6a98d095d451c9027c26b5c1fe429cca89c42 --- /dev/null +++ b/29yeoadw/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdf1c8545f5e547e8f2ed478cebd19c08462c821366938d6636c4bf18e3f19da +size 1240115 diff --git a/29yeoadw/tb/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 b/29yeoadw/tb/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 new file mode 100644 index 0000000000000000000000000000000000000000..69ca5e9772768c584ea5c0666b49fb18d1fda4d6 --- /dev/null +++ b/29yeoadw/tb/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d548c39155376d678d2369ec0d8c68a7a6e4f021e0ceda3ca960c36809726c0 +size 6539 diff --git a/29yeoadw/wandb/debug-internal.log b/29yeoadw/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7890967da49cb1ef32913941b5e7a5181a05730e --- /dev/null +++ b/29yeoadw/wandb/debug-internal.log @@ -0,0 +1,753 @@ +2025-01-04 20:58:20,566 INFO StreamThr :2020231 [internal.py:wandb_internal():85] W&B internal server running at pid: 2020231, started at: 2025-01-04 20:58:20.564958 +2025-01-04 20:58:20,568 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:58:20,579 INFO WriterThread:2020231 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/run-29yeoadw.wandb +2025-01-04 20:58:20,580 DEBUG SenderThread:2020231 [sender.py:send():379] send: header +2025-01-04 20:58:20,587 DEBUG SenderThread:2020231 [sender.py:send():379] send: run +2025-01-04 20:58:20,802 INFO SenderThread:2020231 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files +2025-01-04 20:58:20,802 INFO SenderThread:2020231 [sender.py:_start_run_threads():1188] run started: 29yeoadw with start time 1736042300.564984 +2025-01-04 20:58:20,819 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:58:20,819 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:58:20,896 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:58:20,900 DEBUG HandlerThread:2020231 [system_info.py:__init__():26] System info init +2025-01-04 20:58:20,900 DEBUG HandlerThread:2020231 [system_info.py:__init__():41] System info init done +2025-01-04 20:58:20,901 INFO HandlerThread:2020231 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:58:20,901 INFO SystemMonitor:2020231 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:58:20,901 INFO HandlerThread:2020231 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:58:20,901 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:58:20,902 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:58:20,902 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:58:20,903 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:58:20,904 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:58:20,914 DEBUG HandlerThread:2020231 [system_info.py:probe():152] Probing system +2025-01-04 20:58:20,916 DEBUG HandlerThread:2020231 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_info.py:probe():200] Probing system done +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:58:20.914696', 'startedAt': '2025-01-05T01:58:20.556094', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.98075, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.93, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 637.0136680603027}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:58:20,921 INFO HandlerThread:2020231 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:58:20,921 INFO HandlerThread:2020231 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_info.py:_save_code():44] Saving code +2025-01-04 20:58:20,922 DEBUG HandlerThread:2020231 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:58:20,923 DEBUG HandlerThread:2020231 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:58:20,988 DEBUG HandlerThread:2020231 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:58:20,988 INFO HandlerThread:2020231 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:58:20,991 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 20:58:20,991 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:58:20,992 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:58:20,992 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:58:21,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:58:21,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:58:21,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:58:21,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:58:21,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:21,225 INFO HandlerThread:2020231 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/tb" + save: true +} +control { + relay_id: "166acabc817640e4a5621da54cfafcbb" +} +_info { + stream_id: "29yeoadw" +} + +2025-01-04 20:58:21,225 DEBUG HandlerThread:2020231 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:58:21,236 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/wvc13ify-diff.patch +2025-01-04 20:58:21,239 INFO wandb-upload_1:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/cd8ohl5u-code/FISH/eval_robot.py +2025-01-04 20:58:21,244 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/0nuliayf-wandb-metadata.json +2025-01-04 20:58:21,322 DEBUG SenderThread:2020231 [sender.py:send():379] send: telemetry +2025-01-04 20:58:21,322 DEBUG SenderThread:2020231 [sender.py:send():379] send: tbrecord +2025-01-04 20:58:21,323 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 20:58:21,323 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 with policy live +2025-01-04 20:58:21,501 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/zaaws7nk-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/diff.patch +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-metadata.json +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH/eval_robot.py +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:58:21,805 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH +2025-01-04 20:58:21,805 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code +2025-01-04 20:58:22,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:23,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:23,803 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:24,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:25,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:26,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:26,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:27,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:28,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:29,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:30,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:31,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:31,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:32,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:33,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:34,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:35,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:36,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:58:36,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:58:36,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:36,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:37,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:38,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:39,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:40,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:41,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:42,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:42,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:42,714 DEBUG SenderThread:2020231 [sender.py:send():379] send: config +2025-01-04 20:58:43,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:43,808 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:44,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:45,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:45,809 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:46,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:47,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:47,440 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:47,810 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:48,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:49,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:49,810 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:50,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:51,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:58:51,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:58:51,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:51,811 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:52,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:53,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:53,330 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:53,812 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml +2025-01-04 20:58:54,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:55,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:56,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:57,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:58,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:59,001 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:59,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:59,814 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:00,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:01,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:01,814 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:02,269 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:03,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:03,815 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:04,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:04,269 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:05,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:06,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:06,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:06,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:07,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:08,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:09,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:09,401 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:09,817 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:10,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:11,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:12,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:13,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:14,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:14,565 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:15,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:15,818 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:16,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:17,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:18,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:19,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:20,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:20,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:20,904 DEBUG SystemMonitor:2020231 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:59:20,905 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 20:59:21,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:21,224 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:21,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:22,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:23,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:24,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:25,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:25,297 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:26,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:27,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:28,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:28,606 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 20:59:28,606 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 with policy now +2025-01-04 20:59:28,606 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:59:28,607 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 20:59:28,607 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:59:28,609 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media +2025-01-04 20:59:28,853 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/6o43glor-media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 20:59:29,110 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/6cyiifwz-media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 20:59:29,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:29,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:30,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:30,622 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:31,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:32,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:33,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:33,823 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:34,239 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:35,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:36,077 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:36,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:36,224 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:36,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:36,824 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:59:37,048 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/wqlh8sap-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:59:37,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:37,825 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:38,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:39,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:39,825 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:40,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:41,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:41,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:41,826 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:42,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:43,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:43,827 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:44,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:45,236 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:45,828 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:46,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:46,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:47,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:48,237 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:49,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:50,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:50,910 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 20:59:51,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:51,228 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:51,273 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:51,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:52,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:53,255 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:54,279 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:55,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:56,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:57,240 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:57,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:58,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:59,290 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:00,286 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:01,287 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:02,243 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:02,286 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:03,287 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:04,287 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:05,289 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:06,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:06,225 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:06,290 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:07,288 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:07,333 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:08,290 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:09,288 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:10,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:10,505 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:00:10,506 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:00:10,506 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 with policy now +2025-01-04 21:00:10,506 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:10,506 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:10,507 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:10,768 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/xbfn3pbk-media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:00:10,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:10,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:00:10,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:00:11,119 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/ylgqvcs9-media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:00:11,291 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:11,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:12,291 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:13,244 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:13,291 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:14,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:15,300 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:15,837 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:16,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:17,292 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:18,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:18,329 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:18,640 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:18,640 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:18,647 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:18,838 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:18,839 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:00:19,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:19,839 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:20,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:20,905 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:00:21,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:21,229 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:21,298 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:21,839 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:22,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:23,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:23,689 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:23,840 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:24,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:25,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:26,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:27,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:27,842 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:28,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:29,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:29,669 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:29,842 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:30,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:31,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:32,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:33,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:34,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:35,241 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:35,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:36,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:36,225 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:36,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:37,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:37,845 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:38,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:39,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:40,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:40,474 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:41,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:42,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:43,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:43,847 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:44,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:45,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:45,686 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:46,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:47,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:48,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:49,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:50,297 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:50,533 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:00:50,533 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 with policy now +2025-01-04 21:00:50,535 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:00:50,536 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:50,536 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:50,537 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:50,792 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/be3xc7nl-media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:00:50,849 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:50,850 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:00:50,850 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:00:50,906 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:00:50,906 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:51,091 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/jeqb9oi9-media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:00:51,230 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:51,230 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:51,298 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:51,850 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:52,299 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:53,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:54,309 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:55,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:55,851 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:56,245 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:56,309 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:57,298 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:58,307 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:59,302 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:59,468 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:59,468 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:59,468 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:59,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:59,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:59,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:01:00,091 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/2rcdkycd-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:01:00,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:01,245 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:01,308 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:01,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:02,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:03,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:04,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:05,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:05,855 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:06,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:06,226 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:06,310 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:06,343 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:07,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:08,310 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:09,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:09,856 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:10,312 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:11,312 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:12,243 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:12,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:13,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:14,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:15,312 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:16,313 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:17,313 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:17,773 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:17,859 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:18,320 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:19,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:20,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:20,906 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:01:21,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:21,227 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:21,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:22,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:23,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:23,328 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:24,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:25,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:26,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:27,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:28,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:29,259 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:29,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:30,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:31,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:32,329 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:32,702 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:01:32,703 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 with policy now +2025-01-04 21:01:32,705 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:01:32,706 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:01:32,706 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:01:32,706 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:01:32,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:01:32,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:01:32,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:01:32,994 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/vaovownj-media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:01:33,254 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/efaf1q02-media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:01:33,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:33,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:34,328 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:34,722 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:35,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:36,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:36,235 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:36,344 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:37,336 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:37,866 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:38,334 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:39,332 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:40,281 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:40,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:41,339 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:41,867 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:42,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:43,381 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:43,523 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:01:43,523 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:01:43,523 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:01:43,868 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:01:43,868 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:01:44,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:45,336 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:45,445 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:45,868 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:46,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:47,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:48,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:49,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:49,870 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:50,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:50,907 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:01:50,907 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:51,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:51,228 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:51,339 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:51,871 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:52,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:53,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:54,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:55,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:56,266 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:56,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:57,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:57,873 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:58,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:59,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:00,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:01,266 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:01,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:02,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:03,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:04,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:05,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:06,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:02:06,229 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:02:06,321 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:06,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:07,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:08,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:09,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:10,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:11,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:12,266 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:12,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:13,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:14,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:15,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:16,343 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:17,268 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:17,321 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:02:17,321 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 with policy now +2025-01-04 21:02:17,324 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:02:17,324 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:17,325 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:17,325 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:17,347 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:17,574 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/wz0scwxw-media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:02:18,152 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/dzjdlggs-media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:18,352 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:19,353 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:19,881 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:20,346 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:20,916 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:02:21,237 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:02:21,238 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:02:21,353 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:21,881 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:22,350 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:22,352 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:23,354 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:24,371 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:25,370 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:25,883 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:26,371 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:26,576 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:26,577 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:26,591 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:26,883 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:26,883 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:27,113 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/m018nimf-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:27,368 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:27,390 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:02:27,390 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:02:27,538 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:02:27,841 INFO wandb-upload_2:2020231 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplyet9eov +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval +2025-01-04 21:02:27,962 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:02:27,972 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:02:28,309 INFO SenderThread:2020231 [sender.py:send_request_log_artifact():1525] logged artifact run-29yeoadw-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDUyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MjQyNA==', 'latestArtifact': None}} +2025-01-04 21:02:28,309 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:02:28,309 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:02:28,310 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:28,310 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,310 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:28,310 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,311 DEBUG SenderThread:2020231 [sender.py:send():379] send: telemetry +2025-01-04 21:02:28,311 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,315 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,315 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,315 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,315 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,316 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,316 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,316 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,316 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,316 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,317 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,317 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,317 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,318 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,318 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,318 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,318 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,319 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,319 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,319 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,319 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,320 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,320 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:02:28,367 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:28,483 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/qeanaiml-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:28,729 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/uzeb958t-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:28,744 INFO wandb-upload_0:2020231 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkgj1qts2 +2025-01-04 21:02:28,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:28,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:28,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval +2025-01-04 21:02:29,141 INFO SenderThread:2020231 [sender.py:send_request_log_artifact():1525] logged artifact run-29yeoadw-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDU0Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MjQyNg==', 'latestArtifact': None}} +2025-01-04 21:02:29,141 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:02:29,142 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json with policy now +2025-01-04 21:02:29,142 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:29,142 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:29,142 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:29,327 INFO wandb-upload_1:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/4sz97cmh-media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:29,367 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:29,578 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/chdx1f0w-media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:29,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:29,885 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:30,368 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:30,417 DEBUG SenderThread:2020231 [sender.py:send():379] send: exit +2025-01-04 21:02:30,417 INFO SenderThread:2020231 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:send_exit():588] handling runtime: 249 +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:send_exit():594] send defer +2025-01-04 21:02:30,418 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,418 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:02:30,418 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:02:30,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,419 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:02:30,419 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,419 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:02:30,419 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:02:30,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,419 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:02:30,419 INFO HandlerThread:2020231 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:02:30,419 DEBUG SystemMonitor:2020231 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:02:30,419 DEBUG SystemMonitor:2020231 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:02:30,420 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:02:30,420 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:02:30,471 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:02:30,472 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:02:30,472 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:02:30,472 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,472 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:02:30,472 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:02:30,472 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:02:30,472 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,472 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:02:30,473 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,473 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:02:30,473 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:02:30,473 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,473 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:02:30,885 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:31,885 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:36,581 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:36,581 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:36,581 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:02:36,582 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,582 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:02:36,582 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:02:36,582 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:36,583 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:36,583 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:02:36,583 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:36,583 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:36,584 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:36,584 DEBUG SenderThread:2020231 [sender.py:send():379] send: summary +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:36,588 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:02:36,588 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:36,588 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:02:36,588 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:02:36,592 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:36,734 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:02:36,734 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:36,734 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:02:36,735 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,735 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:02:36,887 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:36,887 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml +2025-01-04 21:02:37,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:37,589 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:02:37,589 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:37,589 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:37,589 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:02:37,590 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:37,590 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:02:37,590 INFO SenderThread:2020231 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:02:37,592 INFO SenderThread:2020231 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:02:37,594 INFO SenderThread:2020231 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:02:37,608 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:02:37,609 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:37,609 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:02:37,609 DEBUG SenderThread:2020231 [sender.py:send():379] send: artifact +2025-01-04 21:02:37,888 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:38,061 INFO wandb-upload_0:2020231 [upload_job.py:push():85] Skipped uploading /tmp/tmpugjbxdvx/wandb-job.json +2025-01-04 21:02:38,062 INFO wandb-upload_2:2020231 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuut5cm5j +2025-01-04 21:02:38,159 INFO wandb-upload_1:2020231 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpetgjqc1f +2025-01-04 21:02:38,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:38,694 INFO SenderThread:2020231 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'versionIndex': 418}}} +2025-01-04 21:02:38,694 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:38,694 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:02:38,694 INFO SenderThread:2020231 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:02:38,888 INFO SenderThread:2020231 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files +2025-01-04 21:02:38,888 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml config.yaml +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json wandb-summary.json +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt requirements.txt +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log output.log +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/diff.patch diff.patch +2025-01-04 21:02:38,895 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:02:38,899 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:38,899 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:38,899 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:02:38,899 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:02:38,900 INFO SenderThread:2020231 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:02:39,079 INFO wandb-upload_1:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt +2025-01-04 21:02:39,114 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml +2025-01-04 21:02:39,120 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:39,131 INFO wandb-upload_4:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:39,197 INFO wandb-upload_3:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/7unxpboo-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:39,397 INFO Thread-11 (_thread_body):2020231 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:02:39,398 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,398 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:02:39,398 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,398 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:02:39,398 INFO SenderThread:2020231 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:02:39,399 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:02:39,399 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,399 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:02:39,400 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,400 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:02:39,400 INFO SenderThread:2020231 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:02:39,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:39,616 INFO SenderThread:2020231 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:02:39,616 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:02:39,617 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:39,617 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,617 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:02:39,617 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,617 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:02:39,617 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:02:39,617 DEBUG SenderThread:2020231 [sender.py:send():379] send: final +2025-01-04 21:02:39,618 DEBUG SenderThread:2020231 [sender.py:send():379] send: footer +2025-01-04 21:02:39,618 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,618 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:02:39,618 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,618 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:02:39,618 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:39,619 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:02:39,619 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:39,619 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:39,619 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:02:39,619 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:02:39,621 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:02:39,622 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:39,673 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:39,673 INFO MainThread:2020231 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:02:39,674 INFO MainThread:2020231 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:02:39,674 INFO MainThread:2020231 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:02:39,674 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:02:39,674 INFO HandlerThread:2020231 [handler.py:finish():882] shutting down handler +2025-01-04 21:02:40,619 INFO WriterThread:2020231 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/run-29yeoadw.wandb +2025-01-04 21:02:40,673 INFO SenderThread:2020231 [sender.py:finish():1615] shutting down sender +2025-01-04 21:02:40,673 INFO SenderThread:2020231 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:02:40,673 INFO SenderThread:2020231 [file_pusher.py:join():175] waiting for file pusher diff --git a/29yeoadw/wandb/debug.log b/29yeoadw/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bf73545e45d5699e7fa0ad12c4dab367520a49d5 --- /dev/null +++ b/29yeoadw/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:58:20,558 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:58:20,558 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Configure stats pid to 2020041 +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/settings +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug.log +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug-internal.log +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw'} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():619] starting backend +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():623] setting up manager +2025-01-04 20:58:20,564 INFO MainThread:2020041 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:58:20,564 INFO MainThread:2020041 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:58:20,579 INFO MainThread:2020041 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:58:20,585 INFO MainThread:2020041 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:58:20,819 INFO MainThread:2020041 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:58:20,890 INFO MainThread:2020041 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:58:20,891 INFO MainThread:2020041 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:58:21,223 INFO MainThread:2020041 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:58:21,224 INFO MainThread:2020041 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:58:21,224 INFO MainThread:2020041 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:58:21,224 INFO MainThread:2020041 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:58:21,225 INFO MainThread:2020041 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:58:21,225 INFO MainThread:2020041 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/tb, True +2025-01-04 20:58:42,713 INFO MainThread:2020041 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 21:02:40,675 WARNING MsgRouterThr:2020041 [router.py:message_loop():77] message_loop has been closed diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH/eval_robot.py b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..011660c271b8c2a23846d3e553df9e8d63a2ea65 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736042300 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/diff.patch b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e6bb84a505bc3c6ddd430d8713d8a91f72ec6d50 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[28, 35, 0.8, 0.6320539712641775, 0.9184053390381375, 0.6375349908792083, 0.901796609082925, 0.6594440936723349, 0.8947329557642636, 0.7148688137101542, 0.8797903916970273]]} \ No newline at end of file diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48d9215418f2da65aa05163acad160d1401de1f3 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:467f8a1696f4f1f555ef1602b3aa06d3d64c6bdfc8557e1daa693e7203a73150 +size 1159188 diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..30849791339a96e6c3438c7db6808aeafe49b112 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b99807ad0b8eaad92e7b7147d72177b57495022891ead586a4e3e37c39171b96 +size 1234058 diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..532f58a228949647305684c7c8fe218b3f08dbb1 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6008c3bedf6112092f657777e0f56a1ee6c9ab080a2d2e2c89bf64d227979595 +size 1166730 diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d549e08b44426fd2a2d12f0209a0938a546f57bb --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f38e890841ba060d24fd1960b3b7f0da23b9be2a3a5464578984e1271c3566a +size 1213103 diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4e6a98d095d451c9027c26b5c1fe429cca89c42 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdf1c8545f5e547e8f2ed478cebd19c08462c821366938d6636c4bf18e3f19da +size 1240115 diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b2ed02604f48ba8795f698c0e2f1f26af9325a6d --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log @@ -0,0 +1,275 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736042322.714210]: resetting environment +[INFO] [1736042322.717067]: cleared current plan +[INFO] [1736042322.718300]: moving to home +[INFO] [1736042323.720457]: reached home +[INFO] [1736042323.722059]: reset action history +[INFO] [1736042326.430949]: environment reset +Starting episode 0 +[INFO] [1736042326.432809]: resetting environment +[INFO] [1736042326.436513]: cleared current plan +[INFO] [1736042326.437577]: moving to home +[INFO] [1736042327.439809]: reached home +[INFO] [1736042327.441380]: reset action history +[INFO] [1736042330.150156]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736042336.953530]: resetting environment +[INFO] [1736042336.956919]: cleared current plan +[INFO] [1736042336.957892]: moving to home +[INFO] [1736042337.960618]: reached home +[INFO] [1736042337.961876]: reset action history +[INFO] [1736042340.669705]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/episode_rosbags/episode_0_2025-01-04-20-59-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2020041.log +started roslaunch server http://158.130.50.37:41725/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2020342] +process[print_text-2]: started with pid [2020365] +process[rosbag_record-3]: started with pid [2020367] +started bagging! +For topic gripper_width: timestamp difference is 100068126 for nearest: 1736042342218228374 - target: 1736042342118160248 at idx 23 +For topic gripper_width: timestamp difference is 105062038 for nearest: 1736042348351514370 - target: 1736042348246452332 at idx 110 +For topic gripper_width: timestamp difference is 109339125 for nearest: 1736042353518287308 - target: 1736042353408948183 at idx 188 +[INFO] [1736042367.690071]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736042368.397840]: Stored episode 1. +[INFO] [1736042368.398113]: Saving video... +[INFO] [1736042368.590184]: Video saved! +Starting episode 1 +[INFO] [1736042368.613739]: resetting environment +[INFO] [1736042368.621617]: cleared current plan +[INFO] [1736042368.622046]: moving to home +[INFO] [1736042372.324623]: reached home +[INFO] [1736042372.325055]: reset action history +[INFO] [1736042375.033942]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042376.680543]: resetting environment +[INFO] [1736042376.688853]: cleared current plan +[INFO] [1736042376.689424]: moving to home +[INFO] [1736042377.691681]: reached home +[INFO] [1736042377.692107]: reset action history +[INFO] [1736042380.399262]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/episode_rosbags/episode_1_2025-01-04-20-59-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2020041.log +started roslaunch server http://158.130.50.37:37385/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2020498] +process[print_text-5]: started with pid [2020499] +process[rosbag_record-6]: started with pid [2020500] +started bagging! +For topic gripper_width: timestamp difference is 101390044 for nearest: 1736042383618189256 - target: 1736042383516799212 at idx 59 +For topic gripper_width: timestamp difference is 102957063 for nearest: 1736042384218551688 - target: 1736042384115594625 at idx 68 +[INFO] [1736042409.398647]: Storing episode... +[pub_text-4] killing on exit +[print_text-5] killing on exit[rosbag_record-6] killing on exit +[INFO] [1736042410.236663]: Stored episode 2. +[INFO] [1736042410.239345]: Saving video... +[INFO] [1736042410.459848]: Video saved! +Starting episode 2 +[INFO] [1736042410.484826]: resetting environment +[INFO] [1736042410.492655]: cleared current plan +[INFO] [1736042410.494022]: moving to home +[INFO] [1736042414.203343]: reached home +[INFO] [1736042414.210347]: reset action history +[INFO] [1736042416.981904]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042418.351699]: resetting environment +[INFO] [1736042418.361669]: cleared current plan +[INFO] [1736042418.362843]: moving to home +[INFO] [1736042419.371020]: reached home +[INFO] [1736042419.372056]: reset action history +[INFO] [1736042422.138524]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/episode_rosbags/episode_2_2025-01-04-21-00-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2020041.log +started roslaunch server http://158.130.50.37:42711/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2020648] +process[print_text-8]: started with pid [2020649] +process[rosbag_record-9]: started with pid [2020650] +started bagging! +For topic gripper_width: timestamp difference is 101909465 for nearest: 1736042425618229455 - target: 1736042425516319990 at idx 51 +For topic gripper_width: timestamp difference is 102345987 for nearest: 1736042425818230673 - target: 1736042425715884686 at idx 53 +For topic gripper_width: timestamp difference is 102748237 for nearest: 1736042426018248639 - target: 1736042425915500402 at idx 55 +For topic gripper_width: timestamp difference is 104374891 for nearest: 1736042426818223243 - target: 1736042426713848352 at idx 64 +For topic gripper_width: timestamp difference is 105977202 for nearest: 1736042427618211890 - target: 1736042427512234688 at idx 77 +For topic gripper_width: timestamp difference is 106401849 for nearest: 1736042436418229270 - target: 1736042436311827421 at idx 194 +For topic gripper_width: timestamp difference is 117027073 for nearest: 1736042441618227749 - target: 1736042441501200676 at idx 249 +[INFO] [1736042449.460190]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736042450.274681]: Stored episode 3. +[INFO] [1736042450.280471]: Saving video... +[INFO] [1736042450.486656]: Video saved! +Starting episode 3 +[INFO] [1736042450.537087]: resetting environment +[INFO] [1736042450.553271]: cleared current plan +[INFO] [1736042450.556943]: moving to home +[INFO] [1736042454.261982]: reached home +[INFO] [1736042454.263610]: reset action history +[INFO] [1736042457.166196]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042459.217558]: resetting environment +[INFO] [1736042459.242726]: cleared current plan +[INFO] [1736042459.247443]: moving to home +[INFO] [1736042460.249595]: reached home +[INFO] [1736042460.250840]: reset action history +[INFO] [1736042463.032879]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/episode_rosbags/episode_3_2025-01-04-21-01-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2020041.log +started roslaunch server http://158.130.50.37:42939/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2020781] +process[print_text-11]: started with pid [2020782] +process[rosbag_record-12]: started with pid [2020783] +started bagging! +For topic gripper_width: timestamp difference is 103338082 for nearest: 1736042468218231995 - target: 1736042468114893913 at idx 65 +For topic gripper_width: timestamp difference is 108827775 for nearest: 1736042476718180125 - target: 1736042476609352350 at idx 195 +[INFO] [1736042491.602791]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1736042492.394542]: Stored episode 4. +[INFO] [1736042492.402687]: Saving video... +[INFO] [1736042492.653629]: Video saved! +Starting episode 4 +[INFO] [1736042492.707899]: resetting environment +[INFO] [1736042492.720005]: cleared current plan +[INFO] [1736042492.720775]: moving to home +[INFO] [1736042496.427736]: reached home +[INFO] [1736042496.438758]: reset action history +[INFO] [1736042499.236037]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042503.376279]: resetting environment +[INFO] [1736042503.392672]: cleared current plan +[INFO] [1736042503.393845]: moving to home +[INFO] [1736042504.396107]: reached home +[INFO] [1736042504.405505]: reset action history +[INFO] [1736042507.200638]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/episode_rosbags/episode_4_2025-01-04-21-01-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2020041.log +started roslaunch server http://158.130.50.37:34331/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2020927] +process[print_text-14]: started with pid [2020928] +process[rosbag_record-15]: started with pid [2020929] +started bagging! +For topic gripper_width: timestamp difference is 102487825 for nearest: 1736042510018179678 - target: 1736042509915691853 at idx 44 +For topic gripper_width: timestamp difference is -102370409 for nearest: 1736042515451512190 - target: 1736042515553882599 at idx 111 +[INFO] [1736042536.228488]: Storing episode... +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[INFO] [1736042537.000346]: Stored episode 5. +[INFO] [1736042537.005753]: Saving video... +[INFO] [1736042537.267451]: Video saved! +[INFO] [1736042537.324914]: resetting environment +[INFO] [1736042537.342390]: cleared current plan +[INFO] [1736042537.344749]: moving to home +[INFO] [1736042541.055116]: reached home +[INFO] [1736042541.058145]: reset action history +[INFO] [1736042543.867874]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-metadata.json b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9748a94d7fb8cf1efaf899bcc54fbb2f227f040b --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T01:58:20.914696", + "startedAt": "2025-01-05T01:58:20.556094", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.98075, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.93, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 637.0136680603027 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d68c67248fd8e44c075e67fc5f962075c9c0cb15 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "467f8a1696f4f1f555ef1602b3aa06d3d64c6bdfc8557e1daa693e7203a73150", "size": 1159188, "path": "media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4"}, "global_step": 665, "_timestamp": 1736042547.8535376, "_runtime": 247.28855347633362, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b99807ad0b8eaad92e7b7147d72177b57495022891ead586a4e3e37c39171b96", "size": 1234058, "path": "media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4"}, "eval/num_success": 5.0, "episode": 35.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "6008c3bedf6112092f657777e0f56a1ee6c9ab080a2d2e2c89bf64d227979595", "size": 1166730, "path": "media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "6f38e890841ba060d24fd1960b3b7f0da23b9be2a3a5464578984e1271c3566a", "size": 1213103, "path": "media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "cdf1c8545f5e547e8f2ed478cebd19c08462c821366938d6636c4bf18e3f19da", "size": 1240115, "path": "media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://8ytrfbjw421pfoh82249vceee6opuimdrc2nq7a4qsouewx8qquv29bqtygpfll88hstv9xi26a60840w1r9qi0ymd7pymi1bn15a5lze0ookhjsadk2jcbgdo93jgcr/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://3hu5fs2uoafwpsfm30tf9b73f11cug0chbo58f4k9dq30dmjdy3x3j0by5buraeip2iq3ccast99uys0joc2xzjqjlvud9ysx8gtnb8qquuqvr24nu4k3891sfy5c2zq:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 28, "total_num_episodes": 35, "total_success_rate": 0.8, "total_success_rate_lb_0.01": 0.6320539712641775, "total_success_rate_ub_0.01": 0.9184053390381375, "total_success_rate_lb_0.025": 0.6375349908792083, "total_success_rate_ub_0.025": 0.901796609082925, "total_success_rate_lb_0.05": 0.6594440936723349, "total_success_rate_ub_0.05": 0.8947329557642636, "total_success_rate_lb_0.1": 0.7148688137101542, "total_success_rate_ub_0.1": 0.8797903916970273, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "fab54c45464571d89cb48578b207cf5ebb04c881abc566018e7a4a14952493a5", "size": 504, "artifact_path": "wandb-client-artifact://ym2t1487dw34b277fbsihlgajz2s6hgypqwios87f9vkcgre2ih21348rzj0y9rnx7754lyts7ccnloabeos6dz3nu60s4watp8p6d12xk5awwnp4r56jp4ahjbpx01c/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vp81tq274nox1m61zoa1shizjgqyyvamejeiwek6fh6v37ktxkvuf8ks61ysvyntj0mupqs08q1gdrjx4agfmc28w3fhcgc0autlit2li59sz6szr7g0oxbhqh8skhi7:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 249}, "eval/total_success_rate": 0.800000011920929, "eval/total_success_rate_lb_0.01": 0.6320539712905884, "eval/total_success_rate_ub_0.01": 0.9184053540229797, "eval/total_success_rate_lb_0.025": 0.6375349760055542, "eval/total_success_rate_ub_0.025": 0.9017965793609619, "eval/total_success_rate_lb_0.05": 0.6594440937042236, "eval/total_success_rate_ub_0.05": 0.8947329521179199, "eval/total_success_rate_lb_0.1": 0.7148687839508057, "eval/total_success_rate_ub_0.1": 0.8797903656959534} \ No newline at end of file diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug-internal.log b/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7890967da49cb1ef32913941b5e7a5181a05730e --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug-internal.log @@ -0,0 +1,753 @@ +2025-01-04 20:58:20,566 INFO StreamThr :2020231 [internal.py:wandb_internal():85] W&B internal server running at pid: 2020231, started at: 2025-01-04 20:58:20.564958 +2025-01-04 20:58:20,568 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:58:20,579 INFO WriterThread:2020231 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/run-29yeoadw.wandb +2025-01-04 20:58:20,580 DEBUG SenderThread:2020231 [sender.py:send():379] send: header +2025-01-04 20:58:20,587 DEBUG SenderThread:2020231 [sender.py:send():379] send: run +2025-01-04 20:58:20,802 INFO SenderThread:2020231 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files +2025-01-04 20:58:20,802 INFO SenderThread:2020231 [sender.py:_start_run_threads():1188] run started: 29yeoadw with start time 1736042300.564984 +2025-01-04 20:58:20,819 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:58:20,819 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:58:20,896 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:58:20,900 DEBUG HandlerThread:2020231 [system_info.py:__init__():26] System info init +2025-01-04 20:58:20,900 DEBUG HandlerThread:2020231 [system_info.py:__init__():41] System info init done +2025-01-04 20:58:20,901 INFO HandlerThread:2020231 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:58:20,901 INFO SystemMonitor:2020231 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:58:20,901 INFO HandlerThread:2020231 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:58:20,901 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:58:20,902 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:58:20,902 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:58:20,903 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:58:20,904 INFO SystemMonitor:2020231 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:58:20,914 DEBUG HandlerThread:2020231 [system_info.py:probe():152] Probing system +2025-01-04 20:58:20,916 DEBUG HandlerThread:2020231 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_info.py:probe():200] Probing system done +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:58:20.914696', 'startedAt': '2025-01-05T01:58:20.556094', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.98075, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.93, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 637.0136680603027}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:58:20,921 INFO HandlerThread:2020231 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:58:20,921 INFO HandlerThread:2020231 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:58:20,921 DEBUG HandlerThread:2020231 [system_info.py:_save_code():44] Saving code +2025-01-04 20:58:20,922 DEBUG HandlerThread:2020231 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:58:20,923 DEBUG HandlerThread:2020231 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:58:20,988 DEBUG HandlerThread:2020231 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:58:20,988 INFO HandlerThread:2020231 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:58:20,991 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 20:58:20,991 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:58:20,992 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:58:20,992 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:58:21,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:58:21,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:58:21,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:58:21,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:58:21,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:21,225 INFO HandlerThread:2020231 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/tb" + save: true +} +control { + relay_id: "166acabc817640e4a5621da54cfafcbb" +} +_info { + stream_id: "29yeoadw" +} + +2025-01-04 20:58:21,225 DEBUG HandlerThread:2020231 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:58:21,236 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/wvc13ify-diff.patch +2025-01-04 20:58:21,239 INFO wandb-upload_1:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/cd8ohl5u-code/FISH/eval_robot.py +2025-01-04 20:58:21,244 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/0nuliayf-wandb-metadata.json +2025-01-04 20:58:21,322 DEBUG SenderThread:2020231 [sender.py:send():379] send: telemetry +2025-01-04 20:58:21,322 DEBUG SenderThread:2020231 [sender.py:send():379] send: tbrecord +2025-01-04 20:58:21,323 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 20:58:21,323 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 with policy live +2025-01-04 20:58:21,501 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/zaaws7nk-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/diff.patch +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-metadata.json +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH/eval_robot.py +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:21,804 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:58:21,805 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH +2025-01-04 20:58:21,805 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code +2025-01-04 20:58:22,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:23,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:23,803 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:24,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:25,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:26,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:26,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:27,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:28,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:29,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:30,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:31,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:31,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:32,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:33,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:34,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:35,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:36,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:58:36,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:58:36,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:36,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:37,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:38,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:39,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:40,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:41,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:42,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:42,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:42,714 DEBUG SenderThread:2020231 [sender.py:send():379] send: config +2025-01-04 20:58:43,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:43,808 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:44,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:45,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:45,809 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:46,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:47,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:47,440 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:47,810 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:48,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:49,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:49,810 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:50,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:51,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:58:51,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:58:51,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:51,811 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:58:52,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:53,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:53,330 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:53,812 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml +2025-01-04 20:58:54,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:55,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:56,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:57,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:58,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:59,001 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:58:59,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:58:59,814 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:00,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:01,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:01,814 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:02,269 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:03,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:03,815 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:04,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:04,269 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:05,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:06,223 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:06,223 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:06,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:07,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:08,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:09,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:09,401 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:09,817 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:10,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:11,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:12,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:13,231 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:14,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:14,565 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:15,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:15,818 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:16,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:17,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:18,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:19,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:20,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:20,232 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:20,904 DEBUG SystemMonitor:2020231 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:59:20,905 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 20:59:21,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:21,224 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:21,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:22,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:23,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:24,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:25,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:25,297 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:26,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:27,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:28,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:28,606 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 20:59:28,606 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 with policy now +2025-01-04 20:59:28,606 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:59:28,607 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 20:59:28,607 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:59:28,609 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 20:59:28,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media +2025-01-04 20:59:28,853 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/6o43glor-media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 20:59:29,110 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/6cyiifwz-media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 20:59:29,233 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:29,822 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:30,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:30,622 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:31,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:32,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:33,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:33,823 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:34,239 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:35,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:36,077 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:36,224 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:36,224 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:36,265 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:36,824 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:59:37,048 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/wqlh8sap-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 20:59:37,234 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:37,825 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:38,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:39,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:39,825 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:40,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:41,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:41,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:41,826 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:42,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:43,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:43,827 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:44,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:45,236 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:45,828 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 20:59:46,227 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:46,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:47,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:48,237 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:49,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:50,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:50,910 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 20:59:51,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:59:51,228 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:59:51,273 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:51,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:52,238 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:53,255 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:54,279 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:55,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:56,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:57,240 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:59:57,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:58,270 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:59:59,290 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:00,286 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:01,287 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:02,243 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:02,286 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:03,287 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:04,287 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:05,289 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:06,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:06,225 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:06,290 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:07,288 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:07,333 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:08,290 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:09,288 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:10,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:10,505 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:00:10,506 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:00:10,506 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 with policy now +2025-01-04 21:00:10,506 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:10,506 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:10,507 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:10,768 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/xbfn3pbk-media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:00:10,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:10,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:00:10,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:00:11,119 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/ylgqvcs9-media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:00:11,291 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:11,836 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:12,291 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:13,244 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:13,291 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:14,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:15,300 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:15,837 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:16,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:17,292 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:18,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:18,329 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:18,640 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:18,640 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:18,647 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:18,838 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:18,839 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:00:19,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:19,839 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:20,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:20,905 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:00:21,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:21,229 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:21,298 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:21,839 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:22,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:23,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:23,689 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:23,840 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:24,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:25,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:26,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:27,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:27,842 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:28,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:29,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:29,669 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:29,842 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:30,293 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:31,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:32,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:33,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:34,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:35,241 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:35,294 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:36,225 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:36,225 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:36,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:37,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:37,845 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:38,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:39,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:40,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:40,474 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:41,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:42,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:43,295 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:43,847 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:44,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:45,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:45,686 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:46,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:47,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:48,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:49,296 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:50,297 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:50,533 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:00:50,533 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 with policy now +2025-01-04 21:00:50,535 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:00:50,536 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:50,536 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:50,537 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:50,792 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/be3xc7nl-media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:00:50,849 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:50,850 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:00:50,850 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:00:50,906 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:00:50,906 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:51,091 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/jeqb9oi9-media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:00:51,230 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:00:51,230 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:00:51,298 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:51,850 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:52,299 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:53,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:54,309 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:55,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:55,851 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:56,245 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:00:56,309 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:57,298 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:58,307 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:59,302 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:00:59,468 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:00:59,468 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:00:59,468 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:00:59,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:00:59,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:00:59,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:01:00,091 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/2rcdkycd-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:01:00,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:01,245 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:01,308 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:01,853 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:02,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:03,304 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:04,305 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:05,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:05,855 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:06,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:06,226 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:06,310 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:06,343 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:07,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:08,310 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:09,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:09,856 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:10,312 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:11,312 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:12,243 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:12,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:13,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:14,311 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:15,312 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:16,313 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:17,313 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:17,773 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:17,859 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:18,320 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:19,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:20,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:20,906 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:01:21,226 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:21,227 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:21,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:22,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:23,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:23,328 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:24,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:25,322 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:26,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:27,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:28,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:29,259 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:29,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:30,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:31,323 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:32,329 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:32,702 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:01:32,703 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 with policy now +2025-01-04 21:01:32,705 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:01:32,706 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:01:32,706 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:01:32,706 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:01:32,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:01:32,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:01:32,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:01:32,994 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/vaovownj-media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:01:33,254 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/efaf1q02-media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:01:33,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:33,864 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:34,328 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:34,722 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:35,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:36,235 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:36,235 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:36,344 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:37,336 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:37,866 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:38,334 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:39,332 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:40,281 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:40,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:41,339 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:41,867 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:42,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:43,381 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:43,523 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:01:43,523 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:01:43,523 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:01:43,868 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:01:43,868 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:01:44,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:45,336 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:45,445 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:45,868 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:46,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:47,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:48,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:49,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:49,870 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:50,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:50,907 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:01:50,907 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:51,228 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:01:51,228 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:01:51,339 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:51,871 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:52,338 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:53,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:54,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:55,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:56,266 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:01:56,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:57,340 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:57,873 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:01:58,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:01:59,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:00,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:01,266 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:01,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:02,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:03,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:04,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:05,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:06,229 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:02:06,229 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:02:06,321 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:06,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:07,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:08,341 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:09,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:10,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:11,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:12,266 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:12,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:13,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:14,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:15,342 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:16,343 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:17,268 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:17,321 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:02:17,321 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 with policy now +2025-01-04 21:02:17,324 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:02:17,324 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:17,325 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:17,325 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:17,347 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:17,574 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/wz0scwxw-media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:17,880 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval +2025-01-04 21:02:18,152 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/dzjdlggs-media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:18,352 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:19,353 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:19,881 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:20,346 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:20,916 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:02:21,237 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:02:21,238 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:02:21,353 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:21,881 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:22,350 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:22,352 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:23,354 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:24,371 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:25,370 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:25,883 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:26,371 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:26,576 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:26,577 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:26,591 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:26,883 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:26,883 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:27,113 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/m018nimf-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:27,368 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:27,390 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:02:27,390 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:02:27,538 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:02:27,841 INFO wandb-upload_2:2020231 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplyet9eov +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table +2025-01-04 21:02:27,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval +2025-01-04 21:02:27,962 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:02:27,972 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:02:28,309 INFO SenderThread:2020231 [sender.py:send_request_log_artifact():1525] logged artifact run-29yeoadw-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDUyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MjQyNA==', 'latestArtifact': None}} +2025-01-04 21:02:28,309 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:02:28,309 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:02:28,310 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:28,310 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,310 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:28,310 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,311 DEBUG SenderThread:2020231 [sender.py:send():379] send: telemetry +2025-01-04 21:02:28,311 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,315 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,315 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,315 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,315 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,316 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,316 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,316 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,316 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,316 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,317 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,317 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,317 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,318 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,318 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,318 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,318 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,319 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,319 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,319 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,319 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:28,320 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:28,320 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:02:28,367 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:28,483 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/qeanaiml-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:28,729 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/uzeb958t-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:28,744 INFO wandb-upload_0:2020231 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkgj1qts2 +2025-01-04 21:02:28,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:28,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:28,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval +2025-01-04 21:02:29,141 INFO SenderThread:2020231 [sender.py:send_request_log_artifact():1525] logged artifact run-29yeoadw-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDU0Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MjQyNg==', 'latestArtifact': None}} +2025-01-04 21:02:29,141 DEBUG SenderThread:2020231 [sender.py:send():379] send: files +2025-01-04 21:02:29,142 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json with policy now +2025-01-04 21:02:29,142 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:29,142 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:29,142 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:29,327 INFO wandb-upload_1:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/4sz97cmh-media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:29,367 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:29,578 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/chdx1f0w-media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:29,884 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:29,885 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:30,368 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:30,417 DEBUG SenderThread:2020231 [sender.py:send():379] send: exit +2025-01-04 21:02:30,417 INFO SenderThread:2020231 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:send_exit():588] handling runtime: 249 +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:send_exit():594] send defer +2025-01-04 21:02:30,418 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,418 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:02:30,418 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:02:30,418 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:02:30,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,419 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:02:30,419 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,419 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:02:30,419 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:02:30,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,419 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:02:30,419 INFO HandlerThread:2020231 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:02:30,419 DEBUG SystemMonitor:2020231 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:02:30,419 DEBUG SystemMonitor:2020231 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:02:30,420 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:02:30,420 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:02:30,471 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:02:30,472 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:02:30,472 INFO HandlerThread:2020231 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:02:30,472 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,472 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:02:30,472 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:02:30,472 DEBUG SenderThread:2020231 [sender.py:send():379] send: stats +2025-01-04 21:02:30,472 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,472 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:02:30,473 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:30,473 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:02:30,473 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:02:30,473 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:30,473 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:02:30,885 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:31,885 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:36,581 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:36,581 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:36,581 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:02:36,582 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,582 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:02:36,582 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:02:36,582 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:36,583 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:36,583 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:02:36,583 DEBUG SenderThread:2020231 [sender.py:send():379] send: history +2025-01-04 21:02:36,583 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:02:36,584 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:36,584 DEBUG SenderThread:2020231 [sender.py:send():379] send: summary +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:02:36,588 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:02:36,588 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:36,588 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:02:36,588 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,588 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:02:36,592 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:02:36,734 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:02:36,734 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:36,734 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:02:36,735 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:36,735 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:02:36,887 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:36,887 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml +2025-01-04 21:02:37,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:37,589 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:02:37,589 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:37,589 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:37,589 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:02:37,590 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:37,590 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:02:37,590 INFO SenderThread:2020231 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:02:37,592 INFO SenderThread:2020231 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:02:37,594 INFO SenderThread:2020231 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:02:37,608 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:02:37,609 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:37,609 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:02:37,609 DEBUG SenderThread:2020231 [sender.py:send():379] send: artifact +2025-01-04 21:02:37,888 INFO Thread-12 :2020231 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:38,061 INFO wandb-upload_0:2020231 [upload_job.py:push():85] Skipped uploading /tmp/tmpugjbxdvx/wandb-job.json +2025-01-04 21:02:38,062 INFO wandb-upload_2:2020231 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuut5cm5j +2025-01-04 21:02:38,159 INFO wandb-upload_1:2020231 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpetgjqc1f +2025-01-04 21:02:38,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:38,694 INFO SenderThread:2020231 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'versionIndex': 418}}} +2025-01-04 21:02:38,694 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:38,694 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:02:38,694 INFO SenderThread:2020231 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:02:38,888 INFO SenderThread:2020231 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files +2025-01-04 21:02:38,888 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml config.yaml +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json wandb-summary.json +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt requirements.txt +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log output.log +2025-01-04 21:02:38,889 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/diff.patch diff.patch +2025-01-04 21:02:38,895 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 media/videos/eval/3_eval_3_6f38e890841ba060d24f.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 media/videos/eval/4_eval_4_cdf1c8545f5e547e8f2e.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 media/videos/eval/2_eval_2_6008c3bedf6112092f65.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 media/videos/eval/1_eval_1_b99807ad0b8eaad92e7b.mp4 +2025-01-04 21:02:38,898 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 media/videos/eval/0_eval_0_467f8a1696f4f1f555ef.mp4 +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json media/table/eval/total_success_rate_ci_6_fab54c45464571d89cb4.table.json +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:02:38,899 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:38,899 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:38,899 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:02:38,899 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:38,899 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:02:38,900 INFO SenderThread:2020231 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:02:39,079 INFO wandb-upload_1:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/requirements.txt +2025-01-04 21:02:39,114 INFO wandb-upload_0:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/config.yaml +2025-01-04 21:02:39,120 INFO wandb-upload_2:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/wandb-summary.json +2025-01-04 21:02:39,131 INFO wandb-upload_4:2020231 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/files/output.log +2025-01-04 21:02:39,197 INFO wandb-upload_3:2020231 [upload_job.py:push():130] Uploaded file /tmp/tmpgk7tua8nwandb/7unxpboo-events.out.tfevents.1736042301.leonmkim-ROG-Strix-G15CS-G15CS.2020041.0 +2025-01-04 21:02:39,397 INFO Thread-11 (_thread_body):2020231 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:02:39,398 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,398 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:02:39,398 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,398 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:02:39,398 INFO SenderThread:2020231 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:02:39,399 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:02:39,399 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,399 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:02:39,400 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,400 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:02:39,400 INFO SenderThread:2020231 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:02:39,419 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:39,616 INFO SenderThread:2020231 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:02:39,616 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:02:39,617 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:39,617 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,617 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:02:39,617 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,617 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:02:39,617 INFO SenderThread:2020231 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:02:39,617 DEBUG SenderThread:2020231 [sender.py:send():379] send: final +2025-01-04 21:02:39,618 DEBUG SenderThread:2020231 [sender.py:send():379] send: footer +2025-01-04 21:02:39,618 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:02:39,618 INFO HandlerThread:2020231 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:02:39,618 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: defer +2025-01-04 21:02:39,618 INFO SenderThread:2020231 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:02:39,618 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:39,619 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:02:39,619 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:39,619 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:02:39,619 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:02:39,619 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:02:39,621 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:02:39,622 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:39,673 DEBUG SenderThread:2020231 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:02:39,673 INFO MainThread:2020231 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:02:39,674 INFO MainThread:2020231 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:02:39,674 INFO MainThread:2020231 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:02:39,674 DEBUG HandlerThread:2020231 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:02:39,674 INFO HandlerThread:2020231 [handler.py:finish():882] shutting down handler +2025-01-04 21:02:40,619 INFO WriterThread:2020231 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/run-29yeoadw.wandb +2025-01-04 21:02:40,673 INFO SenderThread:2020231 [sender.py:finish():1615] shutting down sender +2025-01-04 21:02:40,673 INFO SenderThread:2020231 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:02:40,673 INFO SenderThread:2020231 [file_pusher.py:join():175] waiting for file pusher diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug.log b/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bf73545e45d5699e7fa0ad12c4dab367520a49d5 --- /dev/null +++ b/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:58:20,558 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:58:20,558 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Configure stats pid to 2020041 +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/settings +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug.log +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/wandb/run-20250104_205820-29yeoadw/logs/debug-internal.log +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw'} +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():619] starting backend +2025-01-04 20:58:20,559 INFO MainThread:2020041 [wandb_init.py:init():623] setting up manager +2025-01-04 20:58:20,564 INFO MainThread:2020041 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:58:20,564 INFO MainThread:2020041 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:58:20,579 INFO MainThread:2020041 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:58:20,585 INFO MainThread:2020041 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:58:20,819 INFO MainThread:2020041 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:58:20,890 INFO MainThread:2020041 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:58:20,891 INFO MainThread:2020041 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:58:21,223 INFO MainThread:2020041 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:58:21,224 INFO MainThread:2020041 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:58:21,224 INFO MainThread:2020041 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:58:21,224 INFO MainThread:2020041 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:58:21,225 INFO MainThread:2020041 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:58:21,225 INFO MainThread:2020041 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/29yeoadw/tb, True +2025-01-04 20:58:42,713 INFO MainThread:2020041 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 21:02:40,675 WARNING MsgRouterThr:2020041 [router.py:message_loop():77] message_loop has been closed diff --git a/29yeoadw/wandb/run-20250104_205820-29yeoadw/run-29yeoadw.wandb b/29yeoadw/wandb/run-20250104_205820-29yeoadw/run-29yeoadw.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0bcdbed16c48468f329b5bc8b43fc61081faab15 Binary files /dev/null and b/29yeoadw/wandb/run-20250104_205820-29yeoadw/run-29yeoadw.wandb differ diff --git a/6mzw1wsh/.hydra/config.yaml b/6mzw1wsh/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/6mzw1wsh/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/cxddxu7z/.hydra/config.yaml b/cxddxu7z/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/cxddxu7z/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/cxddxu7z/.hydra/hydra.yaml b/cxddxu7z/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6dee84750b45842da89a6aaea257aaa9c90a09fd --- /dev/null +++ b/cxddxu7z/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/cxddxu7z/.hydra/overrides.yaml b/cxddxu7z/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/cxddxu7z/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/cxddxu7z/episode_rosbags/aligned_depth_to_color_K.npy b/cxddxu7z/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/cxddxu7z/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/cxddxu7z/episode_rosbags/cam_tf_world.npy b/cxddxu7z/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/cxddxu7z/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/cxddxu7z/episode_rosbags/color_K.npy b/cxddxu7z/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/cxddxu7z/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/cxddxu7z/episode_rosbags/depth_K.npy b/cxddxu7z/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/cxddxu7z/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/cxddxu7z/episode_rosbags/episode_0_2025-01-04-22-57-56.bag b/cxddxu7z/episode_rosbags/episode_0_2025-01-04-22-57-56.bag new file mode 100644 index 0000000000000000000000000000000000000000..98667a524065e44f8ba6151a0f32b6e6e4023ae4 --- /dev/null +++ b/cxddxu7z/episode_rosbags/episode_0_2025-01-04-22-57-56.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4317e5f426f206d4a59eadcd33dfd0d3f8b47febaa5bd80a74cd604991e920c +size 1410744140 diff --git a/cxddxu7z/episode_rosbags/episode_1_2025-01-04-22-59-32.bag b/cxddxu7z/episode_rosbags/episode_1_2025-01-04-22-59-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..40e2c68d0ef18a3fb5966c1480cfa3c6b26d9605 --- /dev/null +++ b/cxddxu7z/episode_rosbags/episode_1_2025-01-04-22-59-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc90aac5b076991152601960fa0940a8614f68ebcf58e674a28898b89637b905 +size 1412349340 diff --git a/cxddxu7z/episode_rosbags/episode_2_2025-01-04-23-00-34.bag b/cxddxu7z/episode_rosbags/episode_2_2025-01-04-23-00-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..4575d88b779c71260dc80ee59e4bd495c04b011d --- /dev/null +++ b/cxddxu7z/episode_rosbags/episode_2_2025-01-04-23-00-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c634815d9a23c4b935b82c113276eb05ee66a869708149c5d72bf55786a7da +size 1411002303 diff --git a/cxddxu7z/episode_rosbags/episode_4_2025-01-04-23-02-06.bag b/cxddxu7z/episode_rosbags/episode_4_2025-01-04-23-02-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..56bcfbe929eb28c834f8267ce8e995739c534ca0 --- /dev/null +++ b/cxddxu7z/episode_rosbags/episode_4_2025-01-04-23-02-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0561277308f9961f4e2d8a6b5f5c2f36d2ce6d57619532dfa99a9f9a396c63b4 +size 1410818071 diff --git a/cxddxu7z/eval_robot.log b/cxddxu7z/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..9b1c7e1b714af3e336889763872f341759915d5a --- /dev/null +++ b/cxddxu7z/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:57:26,921][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:57:26,924][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/cxddxu7z/eval_video/0_eval.mp4 b/cxddxu7z/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0541926f825f9e023fe57c576c7a88af5dac54a5 --- /dev/null +++ b/cxddxu7z/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2dd8b2025d43ec844b624b330dd020a343a7e7d4e01c7a4b0372cb33f22e51d +size 1167611 diff --git a/cxddxu7z/eval_video/1_eval.mp4 b/cxddxu7z/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22246a2623e500c37976c550849cfd29b0f0d714 --- /dev/null +++ b/cxddxu7z/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e23084f9a22c63629badee83604ac217a2e87883ac28a9800dacac514fea5d9 +size 1197585 diff --git a/cxddxu7z/eval_video/2_eval.mp4 b/cxddxu7z/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8922e8bcd470f701248d75dc005642d66fc08b22 --- /dev/null +++ b/cxddxu7z/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db9e650e70a5bc836144e608622f26965c862bf7fc3dbe97c7c1047584f7a5ff +size 1173246 diff --git a/cxddxu7z/eval_video/3_eval.mp4 b/cxddxu7z/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e47c63e1e5084889d3c051f40ffe5c12c945d08 --- /dev/null +++ b/cxddxu7z/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7d4589e6ff6e133017f4e96a1fb25310310bb5eb83aa2e9b65e828884c625e0 +size 1191785 diff --git a/cxddxu7z/eval_video/4_eval.mp4 b/cxddxu7z/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73be5eb2c3f8778f406726f8dcb768d0dc08b626 --- /dev/null +++ b/cxddxu7z/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76479c82d6b3a303d94d44c7bcfde555493f1cda8ddbd585f5ceef108684354a +size 1226991 diff --git a/cxddxu7z/tb/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 b/cxddxu7z/tb/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 new file mode 100644 index 0000000000000000000000000000000000000000..be397cb06ce6c2c8fcec0538eb9136de5398c98e --- /dev/null +++ b/cxddxu7z/tb/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eaacda7a16a5ae56e542796d8f1d11ae589fd936fd012a65138e9df77af3dc0 +size 6539 diff --git a/cxddxu7z/wandb/debug-internal.log b/cxddxu7z/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7110dc228228ce8c386d7f3adfb66fe7525b085e --- /dev/null +++ b/cxddxu7z/wandb/debug-internal.log @@ -0,0 +1,858 @@ +2025-01-04 22:57:32,887 INFO StreamThr :2043206 [internal.py:wandb_internal():85] W&B internal server running at pid: 2043206, started at: 2025-01-04 22:57:32.886658 +2025-01-04 22:57:32,888 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:57:32,901 INFO WriterThread:2043206 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/run-cxddxu7z.wandb +2025-01-04 22:57:32,901 DEBUG SenderThread:2043206 [sender.py:send():379] send: header +2025-01-04 22:57:32,909 DEBUG SenderThread:2043206 [sender.py:send():379] send: run +2025-01-04 22:57:33,195 INFO SenderThread:2043206 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files +2025-01-04 22:57:33,195 INFO SenderThread:2043206 [sender.py:_start_run_threads():1188] run started: cxddxu7z with start time 1736049452.889483 +2025-01-04 22:57:33,211 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:57:33,211 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:57:33,259 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:57:33,263 DEBUG HandlerThread:2043206 [system_info.py:__init__():26] System info init +2025-01-04 22:57:33,263 DEBUG HandlerThread:2043206 [system_info.py:__init__():41] System info init done +2025-01-04 22:57:33,264 INFO HandlerThread:2043206 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:57:33,264 INFO SystemMonitor:2043206 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:57:33,264 INFO HandlerThread:2043206 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:57:33,264 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:57:33,265 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:57:33,266 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:57:33,266 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:57:33,267 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:57:33,276 DEBUG HandlerThread:2043206 [system_info.py:probe():152] Probing system +2025-01-04 22:57:33,278 DEBUG HandlerThread:2043206 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:57:33,282 DEBUG HandlerThread:2043206 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:57:33,282 DEBUG HandlerThread:2043206 [system_info.py:probe():200] Probing system done +2025-01-04 22:57:33,283 DEBUG HandlerThread:2043206 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:57:33.276960', 'startedAt': '2025-01-05T03:57:32.873222', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4125.00725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.959, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 794.4538459777832}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:57:33,283 INFO HandlerThread:2043206 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:57:33,283 INFO HandlerThread:2043206 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:57:33,283 DEBUG HandlerThread:2043206 [system_info.py:_save_code():44] Saving code +2025-01-04 22:57:33,284 DEBUG HandlerThread:2043206 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:57:33,284 DEBUG HandlerThread:2043206 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:57:33,349 DEBUG HandlerThread:2043206 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:57:33,349 INFO HandlerThread:2043206 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:57:33,352 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 22:57:33,352 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:57:33,352 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:57:33,353 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:57:33,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:57:33,590 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:57:33,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:57:33,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:57:33,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:33,595 INFO HandlerThread:2043206 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/tb" + save: true +} +control { + relay_id: "4aebd20899484f14965d6491ac9117c7" +} +_info { + stream_id: "cxddxu7z" +} + +2025-01-04 22:57:33,596 DEBUG HandlerThread:2043206 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:57:33,623 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/o2zfekei-wandb-metadata.json +2025-01-04 22:57:33,646 INFO wandb-upload_1:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/d9zbyu6k-code/FISH/eval_robot.py +2025-01-04 22:57:33,650 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/nlnqmg44-diff.patch +2025-01-04 22:57:33,691 DEBUG SenderThread:2043206 [sender.py:send():379] send: telemetry +2025-01-04 22:57:33,692 DEBUG SenderThread:2043206 [sender.py:send():379] send: tbrecord +2025-01-04 22:57:33,692 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 22:57:33,692 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 with policy live +2025-01-04 22:57:33,879 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/7o4xv1ed-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH/eval_robot.py +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-metadata.json +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/diff.patch +2025-01-04 22:57:34,198 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code +2025-01-04 22:57:34,198 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH +2025-01-04 22:57:34,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:35,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:36,196 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:36,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:37,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:38,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:38,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:39,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:39,852 DEBUG SenderThread:2043206 [sender.py:send():379] send: config +2025-01-04 22:57:40,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:41,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:42,198 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:42,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:43,613 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:43,614 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:44,625 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:45,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:46,199 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:46,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:47,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:48,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:57:48,590 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:57:48,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:48,676 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:49,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:50,200 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:50,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:51,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:52,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:53,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:54,201 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:54,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:54,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:55,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:56,202 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:56,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:57,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:58,202 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:58,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:59,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:00,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:00,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:01,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:02,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:03,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:03,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:03,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:04,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:05,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:05,973 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:06,205 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:06,205 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml +2025-01-04 22:58:06,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:07,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:08,205 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:08,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:09,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:10,206 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:10,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:11,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:11,973 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:12,207 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:12,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:13,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:14,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:15,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:16,208 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:16,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:17,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:17,766 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:18,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:18,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:18,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:19,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:20,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:21,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:22,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:23,302 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:23,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:24,242 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 22:58:24,242 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 with policy now +2025-01-04 22:58:24,243 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:58:24,244 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 22:58:24,244 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:58:24,244 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:58:24,514 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/231zvmez-media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 22:58:24,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media +2025-01-04 22:58:25,494 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/swbzaryt-media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 22:58:25,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:26,211 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:26,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:27,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:28,405 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:28,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:29,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:30,213 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:30,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:31,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:32,214 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:32,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:33,267 DEBUG SystemMonitor:2043206 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:58:33,267 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 22:58:33,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:33,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:33,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:33,747 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:34,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:35,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:36,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:37,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:38,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:39,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:39,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:40,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:41,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:42,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:43,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:44,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:45,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:45,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:46,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:47,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:48,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:48,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:48,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:49,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:50,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:50,657 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:51,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:52,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:53,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:54,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:55,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:56,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:56,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:57,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:58,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:59,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:00,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:01,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:01,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:02,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:03,268 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 22:59:03,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:03,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:03,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:04,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:05,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:06,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:06,712 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:07,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:08,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:09,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:10,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:11,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:12,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:12,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:13,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:14,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:15,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:16,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:17,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:17,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:18,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:18,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:18,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:19,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:20,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:21,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:22,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:22,688 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:23,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:24,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:25,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:26,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:27,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:28,586 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:28,629 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:29,233 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:59:29,478 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/u5rq0vg7-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:59:29,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:30,233 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:30,608 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:31,608 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:32,234 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:32,608 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:33,268 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 22:59:33,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:33,592 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:33,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:33,692 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:34,234 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:34,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:35,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:36,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:37,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:38,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:39,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:39,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:40,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:41,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:42,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:43,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:44,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:44,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:45,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:46,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:47,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:48,239 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:48,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:48,592 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:48,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:49,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:49,709 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:50,239 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:50,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:51,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:52,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:53,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:54,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:55,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:55,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:56,612 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:57,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:58,613 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:59,613 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:00,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:00,424 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:00:00,424 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 with policy now +2025-01-04 23:00:00,427 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:00:00,427 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:00:00,427 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:00:00,428 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:00:00,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:00,685 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/ngncbuy8-media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:00:01,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:00:01,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:00:01,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:00:01,443 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:01,523 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/2aokmbge-media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:00:01,614 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:02,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:02,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:03,248 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:03,269 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:00:03,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:03,594 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:03,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:04,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:05,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:06,249 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:06,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:06,825 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:07,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:08,250 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:08,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:09,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:10,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:11,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:12,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:12,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:13,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:14,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:15,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:16,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:17,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:17,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:18,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:18,593 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:18,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:19,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:20,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:21,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:22,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:22,680 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:23,618 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:24,618 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:25,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:26,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:27,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:28,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:28,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:29,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:30,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:31,257 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:00:31,489 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/cmh55nwq-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:00:31,620 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:32,127 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:00:32,127 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:00:32,127 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:00:32,257 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:00:32,257 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:32,620 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:33,269 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:00:33,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:33,593 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:33,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:33,743 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:34,258 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:34,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:35,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:36,259 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:36,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:37,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:38,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:39,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:39,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:40,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:41,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:42,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:43,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:44,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:44,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:45,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:46,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:47,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:48,262 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:48,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:48,593 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:48,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:49,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:50,263 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:50,522 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:50,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:51,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:52,264 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:52,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:53,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:54,265 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:54,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:55,523 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:55,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:56,266 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:56,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:57,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:58,266 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:58,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:59,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:00,267 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:00,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:01,369 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:01,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:02,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:02,628 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:03,156 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:01:03,156 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 with policy now +2025-01-04 23:01:03,162 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:01:03,163 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:01:03,163 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:01:03,163 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:01:03,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:01:03,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:01:03,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:01:03,270 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:01:03,409 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/mjaywtxd-media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:01:03,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:03,600 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:03,625 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:03,658 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/ff94mc2e-media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:01:04,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:04,632 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:05,269 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:05,634 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:06,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:06,641 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:07,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:08,270 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:08,641 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:09,631 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:10,270 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:10,635 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:11,636 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:12,434 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:12,640 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:13,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:14,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:15,644 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:16,650 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:17,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:17,916 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:01:17,916 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:01:17,917 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:17,917 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:01:18,273 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:01:18,273 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:18,274 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:01:18,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:18,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:18,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:19,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:20,274 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:20,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:21,645 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:22,275 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:22,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:23,513 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:23,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:24,275 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:24,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:25,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:26,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:27,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:28,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:28,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:29,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:30,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:31,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:32,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:33,270 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:01:33,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:33,595 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:33,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:33,757 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:34,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:35,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:36,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:37,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:38,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:39,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:39,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:40,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:41,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:42,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:43,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:44,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:44,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:45,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:46,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:47,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:48,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:48,596 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:48,651 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:49,635 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:49,657 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:49,722 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:01:49,723 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 with policy now +2025-01-04 23:01:49,723 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:01:49,724 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:01:49,724 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:01:49,725 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:01:49,978 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/lqxauugd-media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:01:50,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:01:50,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:50,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:01:50,285 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:01:50,547 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/2l5ysptu-media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:01:50,657 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:51,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:51,659 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:52,285 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:52,654 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:53,656 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:54,286 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:54,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:55,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:55,689 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:56,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:57,656 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:58,287 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:58,660 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:59,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:00,661 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:01,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:01,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:02,667 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:03,271 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:02:03,288 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:03,518 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/o8jzy0y5-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:03,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:03,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:03,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:03,975 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:03,975 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:03,975 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:04,289 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:04,289 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:04,669 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:05,667 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:06,290 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:06,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:06,847 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:07,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:08,290 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:08,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:09,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:10,291 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:10,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:11,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:12,051 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:12,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:13,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:14,293 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:14,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:15,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:16,294 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:16,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:17,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:17,894 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:18,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:18,596 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:18,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:19,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:20,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:21,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:22,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:23,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:23,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:24,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:25,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:26,297 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:26,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:27,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:28,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:29,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:29,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:30,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:31,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:32,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:33,271 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:02:33,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:33,596 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:33,666 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:34,666 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:34,715 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:35,670 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:36,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:36,521 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:02:36,521 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 with policy now +2025-01-04 23:02:36,536 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:02:36,537 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:36,537 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:36,537 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:36,674 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:36,796 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/apu8ov85-media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:02:37,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:37,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:02:37,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:02:37,570 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/ifh6kytt-media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:02:37,673 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:38,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:38,696 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:39,687 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:39,975 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:40,684 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:41,682 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:42,303 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:42,678 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:43,679 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:44,304 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:44,686 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:45,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:45,686 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:46,695 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:47,695 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:48,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:48,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:48,695 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:49,692 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:50,693 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:50,722 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:51,699 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:52,035 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:52,035 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:52,036 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:52,251 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 23:02:52,251 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media +2025-01-04 23:02:52,308 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval +2025-01-04 23:02:52,308 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table +2025-01-04 23:02:52,397 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:02:52,527 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/c39p65gw-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:02:52,705 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:52,780 INFO wandb-upload_2:2043206 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj_wsnzxy +2025-01-04 23:02:52,800 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 23:02:52,811 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:02:53,224 INFO SenderThread:2043206 [sender.py:send_request_log_artifact():1525] logged artifact run-cxddxu7z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxNDM3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjUxOQ==', 'latestArtifact': None}} +2025-01-04 23:02:53,224 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:02:53,224 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 23:02:53,224 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:53,225 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,225 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,225 DEBUG SenderThread:2043206 [sender.py:send():379] send: telemetry +2025-01-04 23:02:53,225 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,226 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,226 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,227 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,228 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,228 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,228 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,228 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,228 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,229 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,229 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,229 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,230 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,230 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,230 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,230 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,231 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,231 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,231 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,231 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,231 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,232 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,232 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval +2025-01-04 23:02:53,456 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/oaurte1e-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:02:53,528 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/wemsyofa-media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:02:53,611 INFO wandb-upload_1:2043206 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvdm_1l96 +2025-01-04 23:02:53,693 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:53,996 INFO SenderThread:2043206 [sender.py:send_request_log_artifact():1525] logged artifact run-cxddxu7z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxNDM4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjUyMA==', 'latestArtifact': None}} +2025-01-04 23:02:53,996 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:02:53,997 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json with policy now +2025-01-04 23:02:53,997 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:53,997 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,998 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:54,049 DEBUG SenderThread:2043206 [sender.py:send():379] send: exit +2025-01-04 23:02:54,049 INFO SenderThread:2043206 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:send_exit():588] handling runtime: 320 +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:send_exit():594] send defer +2025-01-04 23:02:54,050 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,050 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 23:02:54,050 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 1 +2025-01-04 23:02:54,051 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 23:02:54,051 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,051 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 23:02:54,051 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 2 +2025-01-04 23:02:54,051 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 23:02:54,051 DEBUG SystemMonitor:2043206 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 23:02:54,054 DEBUG SystemMonitor:2043206 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 23:02:54,052 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 23:02:54,107 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 23:02:54,107 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 23:02:54,107 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined network monitor +2025-01-04 23:02:54,107 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,107 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 23:02:54,108 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 3 +2025-01-04 23:02:54,108 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:02:54,108 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,108 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 23:02:54,109 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,109 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 23:02:54,109 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 4 +2025-01-04 23:02:54,109 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,109 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 23:02:54,238 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/xe1ryk6g-media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:02:54,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:54,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:56,308 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:03:01,041 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:01,041 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:03:01,041 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 23:03:01,042 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,042 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 23:03:01,042 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 5 +2025-01-04 23:03:01,043 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:01,043 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:03:01,043 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:03:01,043 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:01,043 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 23:03:01,044 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:03:01,044 DEBUG SenderThread:2043206 [sender.py:send():379] send: summary +2025-01-04 23:03:01,048 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:03:01,048 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,048 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 23:03:01,048 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 6 +2025-01-04 23:03:01,048 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:01,048 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 23:03:01,049 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,049 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 23:03:01,052 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:03:01,052 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:01,178 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 7 +2025-01-04 23:03:01,178 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:01,178 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:01,178 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 23:03:01,179 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,179 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 23:03:01,310 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:03:01,310 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml +2025-01-04 23:03:02,052 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:02,601 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 8 +2025-01-04 23:03:02,601 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:02,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:02,601 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 23:03:02,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:02,602 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 23:03:02,602 INFO SenderThread:2043206 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 23:03:02,604 INFO SenderThread:2043206 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 23:03:02,606 INFO SenderThread:2043206 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 23:03:02,621 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 9 +2025-01-04 23:03:02,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:02,621 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 23:03:02,621 DEBUG SenderThread:2043206 [sender.py:send():379] send: artifact +2025-01-04 23:03:02,845 INFO SenderThread:2043206 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 23:03:02,845 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:02,845 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 23:03:02,845 INFO SenderThread:2043206 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 23:03:03,053 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:03,311 INFO SenderThread:2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:03:03,311 INFO SenderThread:2043206 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files +2025-01-04 23:03:03,312 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml config.yaml +2025-01-04 23:03:03,312 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json wandb-summary.json +2025-01-04 23:03:03,312 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:03:03,314 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt requirements.txt +2025-01-04 23:03:03,314 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-metadata.json wandb-metadata.json +2025-01-04 23:03:03,315 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log output.log +2025-01-04 23:03:03,315 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/diff.patch diff.patch +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:03:03,318 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 23:03:03,318 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 10 +2025-01-04 23:03:03,321 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:03,321 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,321 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 23:03:03,321 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 23:03:03,503 INFO wandb-upload_1:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml +2025-01-04 23:03:03,526 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:03:03,537 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/a7z1y1w1-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:03:03,562 INFO wandb-upload_3:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt +2025-01-04 23:03:03,565 INFO wandb-upload_4:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:03:03,766 INFO Thread-11 (_thread_body):2043206 [sender.py:transition_state():614] send defer: 11 +2025-01-04 23:03:03,766 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,766 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 23:03:03,766 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,766 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 23:03:03,766 INFO SenderThread:2043206 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 23:03:03,767 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 12 +2025-01-04 23:03:03,768 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,768 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 23:03:03,768 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,768 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 23:03:03,768 INFO SenderThread:2043206 [file_stream.py:finish():601] file stream finish called +2025-01-04 23:03:03,953 INFO SenderThread:2043206 [file_stream.py:finish():605] file stream finish is done +2025-01-04 23:03:03,953 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 13 +2025-01-04 23:03:03,953 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,954 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,954 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 23:03:03,954 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 14 +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send():379] send: final +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send():379] send: footer +2025-01-04 23:03:03,954 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,954 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,955 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 23:03:03,955 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:03,955 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:03,955 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 23:03:03,956 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:03,956 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 23:03:03,956 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: server_info +2025-01-04 23:03:03,958 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 23:03:03,958 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:03:04,016 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:04,016 INFO MainThread:2043206 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 23:03:04,017 INFO MainThread:2043206 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 23:03:04,017 INFO MainThread:2043206 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 23:03:04,017 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 23:03:04,017 INFO HandlerThread:2043206 [handler.py:finish():882] shutting down handler +2025-01-04 23:03:04,956 INFO WriterThread:2043206 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/run-cxddxu7z.wandb +2025-01-04 23:03:05,016 INFO SenderThread:2043206 [sender.py:finish():1615] shutting down sender +2025-01-04 23:03:05,016 INFO SenderThread:2043206 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 23:03:05,016 INFO SenderThread:2043206 [file_pusher.py:join():175] waiting for file pusher diff --git a/cxddxu7z/wandb/debug.log b/cxddxu7z/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..16035a0311b46fac1afe21bb6520fc9f8f2c6e08 --- /dev/null +++ b/cxddxu7z/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Configure stats pid to 2043031 +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/settings +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug.log +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug-internal.log +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z'} +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():619] starting backend +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():623] setting up manager +2025-01-04 22:57:32,885 INFO MainThread:2043031 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:57:32,888 INFO MainThread:2043031 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:57:32,900 INFO MainThread:2043031 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:57:32,907 INFO MainThread:2043031 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:57:33,210 INFO MainThread:2043031 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:57:33,252 INFO MainThread:2043031 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:57:33,252 INFO MainThread:2043031 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:57:33,591 INFO MainThread:2043031 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:57:33,591 INFO MainThread:2043031 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:57:33,591 INFO MainThread:2043031 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:57:33,592 INFO MainThread:2043031 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:57:33,594 INFO MainThread:2043031 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:57:33,595 INFO MainThread:2043031 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/tb, True +2025-01-04 22:57:39,851 INFO MainThread:2043031 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 23:03:05,018 WARNING MsgRouterThr:2043031 [router.py:message_loop():77] message_loop has been closed diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH/eval_robot.py b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc1c094b583af627c28662cddbf3047e478921aa --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736049452 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/diff.patch b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json new file mode 100644 index 0000000000000000000000000000000000000000..4336f59fbe8cc6fe7f990c9e98e1ca96b32a566c --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[42, 60, 0.7, 0.5615226626273133, 0.8193268030243228, 0.5719725340460837, 0.801160454689829, 0.5897405594407845, 0.7906480207457988, 0.627101436231942, 0.7740943207788641]]} \ No newline at end of file diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0541926f825f9e023fe57c576c7a88af5dac54a5 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2dd8b2025d43ec844b624b330dd020a343a7e7d4e01c7a4b0372cb33f22e51d +size 1167611 diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22246a2623e500c37976c550849cfd29b0f0d714 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e23084f9a22c63629badee83604ac217a2e87883ac28a9800dacac514fea5d9 +size 1197585 diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8922e8bcd470f701248d75dc005642d66fc08b22 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db9e650e70a5bc836144e608622f26965c862bf7fc3dbe97c7c1047584f7a5ff +size 1173246 diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e47c63e1e5084889d3c051f40ffe5c12c945d08 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7d4589e6ff6e133017f4e96a1fb25310310bb5eb83aa2e9b65e828884c625e0 +size 1191785 diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73be5eb2c3f8778f406726f8dcb768d0dc08b626 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76479c82d6b3a303d94d44c7bcfde555493f1cda8ddbd585f5ceef108684354a +size 1226991 diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1b36d536cc14e05caaa6b7bd466976d2827fd770 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log @@ -0,0 +1,294 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736049459.852587]: resetting environment +[INFO] [1736049459.856701]: cleared current plan +[INFO] [1736049459.859511]: moving to home +[INFO] [1736049460.861274]: reached home +[INFO] [1736049460.862889]: reset action history +[INFO] [1736049463.570942]: environment reset +Starting episode 0 +[INFO] [1736049463.573183]: resetting environment +[INFO] [1736049463.576509]: cleared current plan +[INFO] [1736049463.577446]: moving to home +[INFO] [1736049464.579475]: reached home +[INFO] [1736049464.581642]: reset action history +[INFO] [1736049467.290328]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736049472.523717]: resetting environment +[INFO] [1736049472.526796]: cleared current plan +[INFO] [1736049472.527642]: moving to home +[INFO] [1736049473.529348]: reached home +[INFO] [1736049473.530610]: reset action history +[INFO] [1736049476.288251]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/episode_rosbags/episode_0_2025-01-04-22-57-56.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2043031.log +started roslaunch server http://158.130.50.37:33193/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2043306] +process[print_text-2]: started with pid [2043330] +process[rosbag_record-3]: started with pid [2043331] +started bagging! +For topic gripper_width: timestamp difference is 102597711 for nearest: 1736049483718156812 - target: 1736049483615559101 at idx 100 +For topic gripper_width: timestamp difference is 100446002 for nearest: 1736049484918157355 - target: 1736049484817711353 at idx 118 +For topic gripper_width: timestamp difference is -105270675 for nearest: 1736049486951504895 - target: 1736049487056775570 at idx 142 +For topic gripper_width: timestamp difference is -104430227 for nearest: 1736049487351491661 - target: 1736049487455921888 at idx 147 +For topic gripper_width: timestamp difference is 102311510 for nearest: 1736049490918117660 - target: 1736049490815806150 at idx 195 +For topic gripper_width: timestamp difference is 107884454 for nearest: 1736049493718159484 - target: 1736049493610275030 at idx 234 +[INFO] [1736049503.301665]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736049504.035965]: Stored episode 1. +[INFO] [1736049504.037590]: Saving video... +[INFO] [1736049504.232047]: Video saved! +Starting episode 1 +[INFO] [1736049504.243866]: resetting environment +[INFO] [1736049504.258207]: cleared current plan +[INFO] [1736049504.259625]: moving to home +[INFO] [1736049507.361212]: reached home +[INFO] [1736049507.361668]: reset action history +[INFO] [1736049510.070311]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049568.590264]: resetting environment +[INFO] [1736049568.592339]: cleared current plan +[INFO] [1736049568.592704]: moving to home +[INFO] [1736049569.595558]: reached home +[INFO] [1736049569.596891]: reset action history +[INFO] [1736049572.303560]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/episode_rosbags/episode_1_2025-01-04-22-59-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2043031.log +started roslaunch server http://158.130.50.37:34497/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2043467] +process[print_text-5]: started with pid [2043468] +process[rosbag_record-6]: started with pid [2043492] +started bagging! +For topic gripper_width: timestamp difference is 118897174 for nearest: 1736049586518156026 - target: 1736049586399258852 at idx 188 +For topic gripper_width: timestamp difference is 120167613 for nearest: 1736049588118153453 - target: 1736049587997985840 at idx 210 +[INFO] [1736049599.270424]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736049600.028945]: Stored episode 2. +[INFO] [1736049600.029224]: Saving video... +[INFO] [1736049600.395395]: Video saved! +Starting episode 2 +[INFO] [1736049600.427011]: resetting environment +[INFO] [1736049600.440871]: cleared current plan +[INFO] [1736049600.442489]: moving to home +[WARN] [1736049600.422634]: Plan exhausted +[INFO] [1736049604.049613]: reached home +[INFO] [1736049604.051328]: reset action history +[INFO] [1736049606.821083]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049630.981387]: resetting environment +[INFO] [1736049630.989607]: cleared current plan +[INFO] [1736049630.989840]: moving to home +[INFO] [1736049631.995661]: reached home +[INFO] [1736049631.996190]: reset action history +[INFO] [1736049634.703450]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/episode_rosbags/episode_2_2025-01-04-23-00-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2043031.log +started roslaunch server http://158.130.50.37:39471/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2043617] +process[print_text-8]: started with pid [2043641] +process[rosbag_record-9]: started with pid [2043642] +started bagging! +For topic gripper_width: timestamp difference is 100140544 for nearest: 1736049645918159696 - target: 1736049645818019152 at idx 148 +For topic gripper_width: timestamp difference is 102919201 for nearest: 1736049649318156103 - target: 1736049649215236902 at idx 194 +For topic gripper_width: timestamp difference is 103088916 for nearest: 1736049649518155873 - target: 1736049649415066957 at idx 196 +For topic gripper_width: timestamp difference is 104261177 for nearest: 1736049650918187159 - target: 1736049650813925982 at idx 216 +For topic gripper_width: timestamp difference is 104359457 for nearest: 1736049651118119308 - target: 1736049651013759851 at idx 218 +For topic gripper_width: timestamp difference is 105774833 for nearest: 1736049652918156816 - target: 1736049652812381983 at idx 245 +For topic gripper_width: timestamp difference is 107706258 for nearest: 1736049655318157384 - target: 1736049655210451126 at idx 249 +For topic gripper_width: timestamp difference is 107895153 for nearest: 1736049655518185633 - target: 1736049655410290480 at idx 249 +For topic gripper_width: timestamp difference is 110797470 for nearest: 1736049659118186539 - target: 1736049659007389069 at idx 249 +For topic gripper_width: timestamp difference is 111768177 for nearest: 1736049660318171909 - target: 1736049660206403732 at idx 249 +[INFO] [1736049661.705387]: Storing episode... +[print_text-8] killing on exit[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[INFO] [1736049662.501402]: Stored episode 3. +[INFO] [1736049662.504068]: Saving video... +[INFO] [1736049663.109476]: Video saved! +Starting episode 3 +[INFO] [1736049663.164571]: resetting environment +[INFO] [1736049663.180623]: cleared current plan +[INFO] [1736049663.189594]: moving to home +[WARN] [1736049663.021822]: Plan exhausted +[WARN] [1736049663.066551]: Plan exhausted +[WARN] [1736049663.100253]: Plan exhausted +[WARN] [1736049663.139734]: Plan exhausted +[INFO] [1736049666.596796]: reached home +[INFO] [1736049666.599164]: reset action history +[INFO] [1736049669.392131]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049677.347221]: resetting environment +[INFO] [1736049677.358380]: cleared current plan +[INFO] [1736049677.360355]: moving to home +[INFO] [1736049678.363751]: reached home +[INFO] [1736049678.365108]: reset action history +[INFO] [1736049681.255666]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/episode_rosbags/episode_3_2025-01-04-23-01-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2043031.log +started roslaunch server http://158.130.50.37:45869/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2043752] +process[print_text-11]: started with pid [2043753] +process[rosbag_record-12]: started with pid [2043754] +started bagging! +[INFO] [1736049708.212009]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736049709.014403]: Stored episode 4. +[INFO] [1736049709.018044]: Saving video... +[WARN] [1736049709.393515]: Plan exhausted +[WARN] [1736049709.433536]: Plan exhausted +[WARN] [1736049709.473908]: Plan exhausted +[WARN] [1736049709.513526]: Plan exhausted +[WARN] [1736049709.555003]: Plan exhausted +[WARN] [1736049709.594550]: Plan exhausted +[WARN] [1736049709.634160]: Plan exhausted +[WARN] [1736049709.675574]: Plan exhausted +[WARN] [1736049709.713690]: Plan exhausted +[INFO] [1736049709.682122]: Video saved! +Starting episode 4 +[INFO] [1736049709.725647]: resetting environment +[INFO] [1736049709.740757]: cleared current plan +[INFO] [1736049709.748380]: moving to home +[INFO] [1736049712.853994]: reached home +[INFO] [1736049712.856518]: reset action history +[INFO] [1736049715.642440]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049723.096956]: resetting environment +[INFO] [1736049723.117070]: cleared current plan +[INFO] [1736049723.122473]: moving to home +[INFO] [1736049724.125259]: reached home +[INFO] [1736049724.126148]: reset action history +[INFO] [1736049726.846931]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/episode_rosbags/episode_4_2025-01-04-23-02-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2043031.log +started roslaunch server http://158.130.50.37:43093/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2043922] +process[print_text-14]: started with pid [2043923] +process[rosbag_record-15]: started with pid [2043924] +started bagging! +For topic gripper_width: timestamp difference is -196882851 for nearest: 1736049731751450174 - target: 1736049731948333025 at idx 79 +For topic gripper_width: timestamp difference is 105244100 for nearest: 1736049734851443708 - target: 1736049734746199608 at idx 118 +For topic gripper_width: timestamp difference is 104378222 for nearest: 1736049744251441239 - target: 1736049744147063017 at idx 247 +[INFO] [1736049755.265152]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736049755.997019]: Stored episode 5. +[INFO] [1736049756.006122]: Saving video... +[INFO] [1736049756.462896]: Video saved! +[INFO] [1736049756.536989]: resetting environment +[INFO] [1736049756.556657]: cleared current plan +[INFO] [1736049756.562371]: moving to home +[INFO] [1736049759.971805]: reached home +[INFO] [1736049759.981020]: reset action history +[INFO] [1736049762.815556]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-metadata.json b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c92213efe6df3e79a92d68b29ed73468fabeb60e --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:57:33.276960", + "startedAt": "2025-01-05T03:57:32.873222", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4125.00725, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.004, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.959, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 794.4538459777832 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..38054e428d0b9185c075208745041d830362ebd6 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c2dd8b2025d43ec844b624b330dd020a343a7e7d4e01c7a4b0372cb33f22e51d", "size": 1167611, "path": "media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4"}, "global_step": 663, "_timestamp": 1736049772.7154706, "_runtime": 319.82598757743835, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "7e23084f9a22c63629badee83604ac217a2e87883ac28a9800dacac514fea5d9", "size": 1197585, "path": "media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4"}, "eval/num_success": 0.0, "episode": 60.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "db9e650e70a5bc836144e608622f26965c862bf7fc3dbe97c7c1047584f7a5ff", "size": 1173246, "path": "media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c7d4589e6ff6e133017f4e96a1fb25310310bb5eb83aa2e9b65e828884c625e0", "size": 1191785, "path": "media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "76479c82d6b3a303d94d44c7bcfde555493f1cda8ddbd585f5ceef108684354a", "size": 1226991, "path": "media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://e7yb3lzlib0i82tbdnvrb3i8gkkoo1x756quc6kk698cclxyrm7gehrdo49y7lg472e5n6rrpopvmcnb71izjj361kh44l8gz266r3i6a8emkamvutnk6m2r3r79l3q6/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://gtlh6eyshuj09i8o0r0hgt3ot90nepbx58s0gyty84ezwsmeg1qas4glpxjyhtcgt1oqsbjqvs690wrzdh3qhlfs271iipg8oz3ssmyt2aubt5n35350sqf1tzorqboo:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 42, "total_num_episodes": 60, "total_success_rate": 0.7, "total_success_rate_lb_0.01": 0.5615226626273133, "total_success_rate_ub_0.01": 0.8193268030243228, "total_success_rate_lb_0.025": 0.5719725340460837, "total_success_rate_ub_0.025": 0.801160454689829, "total_success_rate_lb_0.05": 0.5897405594407845, "total_success_rate_ub_0.05": 0.7906480207457988, "total_success_rate_lb_0.1": 0.627101436231942, "total_success_rate_ub_0.1": 0.7740943207788641, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "90f757a4d783c43f5fec7ba35f8c8185dcdd99c3320f1733951abcd4f5cc4a34", "size": 503, "artifact_path": "wandb-client-artifact://q1tu4ubcvbw7vkj6vg58js20r5tlotu09d4jwsj7a8t14nbhb1unibuliarnibzrpdt5ct1owors48s4vqxlneia1a8di70qa5hhkmiiztbm81xwxlq445p278me6t6u/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://j28p3vk0zx4mbuj6elh7sx0s7varlyrftwm2o261cw0mvp663pmhpksr2bvf0tau82m9fgk0doi21zlxrvrae8yrhuvsa5zsca81fyo7w6enckl4x59p4wzzumo14a14:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 320}, "eval/total_success_rate": 0.699999988079071, "eval/total_success_rate_lb_0.01": 0.5615226626396179, "eval/total_success_rate_ub_0.01": 0.8193268179893494, "eval/total_success_rate_lb_0.025": 0.5719725489616394, "eval/total_success_rate_ub_0.025": 0.801160454750061, "eval/total_success_rate_lb_0.05": 0.5897405743598938, "eval/total_success_rate_ub_0.05": 0.7906480431556702, "eval/total_success_rate_lb_0.1": 0.6271014213562012, "eval/total_success_rate_ub_0.1": 0.7740943431854248} \ No newline at end of file diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug-internal.log b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7110dc228228ce8c386d7f3adfb66fe7525b085e --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug-internal.log @@ -0,0 +1,858 @@ +2025-01-04 22:57:32,887 INFO StreamThr :2043206 [internal.py:wandb_internal():85] W&B internal server running at pid: 2043206, started at: 2025-01-04 22:57:32.886658 +2025-01-04 22:57:32,888 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:57:32,901 INFO WriterThread:2043206 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/run-cxddxu7z.wandb +2025-01-04 22:57:32,901 DEBUG SenderThread:2043206 [sender.py:send():379] send: header +2025-01-04 22:57:32,909 DEBUG SenderThread:2043206 [sender.py:send():379] send: run +2025-01-04 22:57:33,195 INFO SenderThread:2043206 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files +2025-01-04 22:57:33,195 INFO SenderThread:2043206 [sender.py:_start_run_threads():1188] run started: cxddxu7z with start time 1736049452.889483 +2025-01-04 22:57:33,211 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:57:33,211 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:57:33,259 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:57:33,263 DEBUG HandlerThread:2043206 [system_info.py:__init__():26] System info init +2025-01-04 22:57:33,263 DEBUG HandlerThread:2043206 [system_info.py:__init__():41] System info init done +2025-01-04 22:57:33,264 INFO HandlerThread:2043206 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:57:33,264 INFO SystemMonitor:2043206 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:57:33,264 INFO HandlerThread:2043206 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:57:33,264 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:57:33,265 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:57:33,266 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:57:33,266 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:57:33,267 INFO SystemMonitor:2043206 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:57:33,276 DEBUG HandlerThread:2043206 [system_info.py:probe():152] Probing system +2025-01-04 22:57:33,278 DEBUG HandlerThread:2043206 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:57:33,282 DEBUG HandlerThread:2043206 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:57:33,282 DEBUG HandlerThread:2043206 [system_info.py:probe():200] Probing system done +2025-01-04 22:57:33,283 DEBUG HandlerThread:2043206 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:57:33.276960', 'startedAt': '2025-01-05T03:57:32.873222', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4125.00725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.959, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 794.4538459777832}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:57:33,283 INFO HandlerThread:2043206 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:57:33,283 INFO HandlerThread:2043206 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:57:33,283 DEBUG HandlerThread:2043206 [system_info.py:_save_code():44] Saving code +2025-01-04 22:57:33,284 DEBUG HandlerThread:2043206 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:57:33,284 DEBUG HandlerThread:2043206 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:57:33,349 DEBUG HandlerThread:2043206 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:57:33,349 INFO HandlerThread:2043206 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:57:33,352 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 22:57:33,352 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:57:33,352 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:57:33,353 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:57:33,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:57:33,590 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:57:33,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:57:33,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:57:33,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:33,595 INFO HandlerThread:2043206 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/tb" + save: true +} +control { + relay_id: "4aebd20899484f14965d6491ac9117c7" +} +_info { + stream_id: "cxddxu7z" +} + +2025-01-04 22:57:33,596 DEBUG HandlerThread:2043206 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:57:33,623 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/o2zfekei-wandb-metadata.json +2025-01-04 22:57:33,646 INFO wandb-upload_1:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/d9zbyu6k-code/FISH/eval_robot.py +2025-01-04 22:57:33,650 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/nlnqmg44-diff.patch +2025-01-04 22:57:33,691 DEBUG SenderThread:2043206 [sender.py:send():379] send: telemetry +2025-01-04 22:57:33,692 DEBUG SenderThread:2043206 [sender.py:send():379] send: tbrecord +2025-01-04 22:57:33,692 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 22:57:33,692 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 with policy live +2025-01-04 22:57:33,879 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/7o4xv1ed-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH/eval_robot.py +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-metadata.json +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:34,197 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/diff.patch +2025-01-04 22:57:34,198 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code +2025-01-04 22:57:34,198 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH +2025-01-04 22:57:34,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:35,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:36,196 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:36,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:37,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:38,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:38,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:39,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:39,852 DEBUG SenderThread:2043206 [sender.py:send():379] send: config +2025-01-04 22:57:40,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:41,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:42,198 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:42,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:43,613 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:43,614 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:44,625 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:45,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:46,199 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:46,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:47,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:48,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:57:48,590 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:57:48,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:48,676 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:49,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:50,200 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:50,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:51,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:52,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:53,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:54,201 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:54,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:54,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:55,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:56,202 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:56,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:57,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:58,202 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:57:58,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:59,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:00,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:00,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:01,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:02,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:03,590 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:03,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:03,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:04,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:05,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:05,973 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:06,205 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:06,205 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml +2025-01-04 22:58:06,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:07,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:08,205 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:08,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:09,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:10,206 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:10,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:11,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:11,973 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:12,207 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:12,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:13,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:14,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:15,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:16,208 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:16,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:17,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:17,766 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:18,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:18,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:18,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:19,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:20,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:21,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:22,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:23,302 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:23,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:24,242 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 22:58:24,242 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 with policy now +2025-01-04 22:58:24,243 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:58:24,244 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 22:58:24,244 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:58:24,244 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:58:24,514 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/231zvmez-media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 22:58:24,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 22:58:25,210 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media +2025-01-04 22:58:25,494 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/swbzaryt-media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 22:58:25,598 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:26,211 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:26,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:27,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:28,405 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:28,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:29,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:30,213 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:30,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:31,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:32,214 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:58:32,599 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:33,267 DEBUG SystemMonitor:2043206 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:58:33,267 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 22:58:33,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:33,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:33,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:33,747 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:34,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:35,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:36,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:37,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:38,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:39,592 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:39,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:40,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:41,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:42,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:43,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:44,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:45,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:45,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:46,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:47,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:48,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:58:48,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:58:48,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:49,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:50,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:50,657 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:51,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:52,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:53,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:54,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:55,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:56,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:58:56,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:57,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:58,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:58:59,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:00,603 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:01,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:01,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:02,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:03,268 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 22:59:03,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:03,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:03,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:04,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:05,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:06,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:06,712 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:07,604 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:08,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:09,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:10,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:11,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:12,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:12,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:13,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:14,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:15,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:16,605 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:17,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:17,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:18,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:18,591 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:18,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:19,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:20,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:21,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:22,606 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:22,688 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:23,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:24,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:25,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:26,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:27,607 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:28,586 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:28,629 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:29,233 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:59:29,478 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/u5rq0vg7-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 22:59:29,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:30,233 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:30,608 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:31,608 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:32,234 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:32,608 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:33,268 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 22:59:33,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:33,592 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:33,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:33,692 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:34,234 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:34,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:35,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:36,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:37,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:38,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:39,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:39,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:40,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:41,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:42,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:43,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:44,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:44,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:45,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:46,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:47,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:48,239 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:48,591 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:59:48,592 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:59:48,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:49,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:49,709 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:50,239 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 22:59:50,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:51,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:52,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:53,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:54,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:55,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:59:55,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:56,612 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:57,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:58,613 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:59:59,613 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:00,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:00,424 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:00:00,424 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 with policy now +2025-01-04 23:00:00,427 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:00:00,427 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:00:00,427 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:00:00,428 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:00:00,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:00,685 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/ngncbuy8-media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:00:01,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:00:01,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:00:01,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:00:01,443 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:01,523 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/2aokmbge-media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:00:01,614 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:02,247 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:02,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:03,248 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:03,269 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:00:03,594 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:03,594 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:03,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:04,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:05,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:06,249 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:06,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:06,825 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:07,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:08,250 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:08,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:09,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:10,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:11,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:12,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:12,615 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:13,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:14,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:15,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:16,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:17,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:17,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:18,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:18,593 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:18,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:19,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:20,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:21,617 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:22,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:22,680 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:23,618 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:24,618 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:25,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:26,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:27,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:28,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:28,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:29,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:30,619 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:31,257 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:00:31,489 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/cmh55nwq-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:00:31,620 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:32,127 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:00:32,127 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:00:32,127 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:00:32,257 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:00:32,257 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:32,620 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:33,269 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:00:33,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:33,593 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:33,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:33,743 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:34,258 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:34,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:35,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:36,259 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:36,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:37,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:38,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:39,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:39,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:40,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:41,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:42,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:43,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:44,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:44,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:45,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:46,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:47,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:48,262 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:48,593 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:00:48,593 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:00:48,637 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:49,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:50,263 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:50,522 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:50,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:51,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:52,264 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:52,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:53,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:54,265 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:54,622 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:55,523 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:00:55,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:56,266 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:56,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:57,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:58,266 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:00:58,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:00:59,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:00,267 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:00,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:01,369 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:01,623 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:02,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:02,628 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:03,156 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:01:03,156 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 with policy now +2025-01-04 23:01:03,162 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:01:03,163 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:01:03,163 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:01:03,163 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:01:03,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:01:03,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:01:03,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:01:03,270 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:01:03,409 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/mjaywtxd-media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:01:03,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:03,600 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:03,625 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:03,658 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/ff94mc2e-media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:01:04,268 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:04,632 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:05,269 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:05,634 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:06,597 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:06,641 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:07,633 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:08,270 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:08,641 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:09,631 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:10,270 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:10,635 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:11,636 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:12,434 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:12,640 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:13,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:14,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:15,644 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:16,650 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:17,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:17,916 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:01:17,916 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:01:17,917 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:17,917 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:01:18,273 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:01:18,273 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:18,274 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:01:18,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:18,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:18,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:19,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:20,274 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:20,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:21,645 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:22,275 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:22,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:23,513 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:23,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:24,275 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:24,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:25,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:26,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:27,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:28,600 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:28,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:29,646 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:30,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:31,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:32,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:33,270 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:01:33,595 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:33,595 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:33,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:33,757 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:34,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:35,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:36,647 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:37,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:38,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:39,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:39,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:40,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:41,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:42,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:43,648 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:44,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:44,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:45,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:46,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:47,649 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:48,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:01:48,596 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:01:48,651 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:49,635 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:49,657 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:49,722 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:01:49,723 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 with policy now +2025-01-04 23:01:49,723 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:01:49,724 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:01:49,724 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:01:49,725 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:01:49,978 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/lqxauugd-media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:01:50,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:01:50,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:50,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:01:50,285 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:01:50,547 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/2l5ysptu-media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:01:50,657 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:51,284 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:51,659 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:52,285 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:52,654 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:53,656 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:54,286 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:54,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:55,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:01:55,689 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:56,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:57,656 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:58,287 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:01:58,660 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:01:59,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:00,661 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:01,611 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:01,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:02,667 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:03,271 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:02:03,288 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:03,518 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/o8jzy0y5-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:03,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:03,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:03,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:03,975 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:03,975 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:03,975 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:04,289 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:04,289 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:04,669 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:05,667 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:06,290 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:06,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:06,847 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:07,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:08,290 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:08,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:09,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:10,291 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:10,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:11,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:12,051 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:12,662 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:13,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:14,293 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:14,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:15,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:16,294 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:16,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:17,663 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:17,894 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:18,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:18,596 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:18,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:19,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:20,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:21,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:22,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:23,610 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:23,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:24,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:25,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:26,297 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:26,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:27,664 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:28,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:29,609 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:29,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:30,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:31,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:32,665 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:33,271 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:02:33,596 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:33,596 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:33,666 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:34,666 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:34,715 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:35,670 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:36,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:36,521 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:02:36,521 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 with policy now +2025-01-04 23:02:36,536 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:02:36,537 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:36,537 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:36,537 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:36,674 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:36,796 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/apu8ov85-media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:02:37,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:37,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:02:37,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval +2025-01-04 23:02:37,570 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/ifh6kytt-media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:02:37,673 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:38,301 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:38,696 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:39,687 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:39,975 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:40,684 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:41,682 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:42,303 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:42,678 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:43,679 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:44,304 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:44,686 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:45,616 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:45,686 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:46,695 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:47,695 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:48,602 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 23:02:48,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: stop_status +2025-01-04 23:02:48,695 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:49,692 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:50,693 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:50,722 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:02:51,699 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:52,035 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:52,035 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:52,036 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:52,251 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 23:02:52,251 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:02:52,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media +2025-01-04 23:02:52,308 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval +2025-01-04 23:02:52,308 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table +2025-01-04 23:02:52,397 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:02:52,527 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/c39p65gw-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:02:52,705 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:52,780 INFO wandb-upload_2:2043206 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj_wsnzxy +2025-01-04 23:02:52,800 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 23:02:52,811 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 23:02:53,224 INFO SenderThread:2043206 [sender.py:send_request_log_artifact():1525] logged artifact run-cxddxu7z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxNDM3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjUxOQ==', 'latestArtifact': None}} +2025-01-04 23:02:53,224 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:02:53,224 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 23:02:53,224 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:53,225 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,225 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,225 DEBUG SenderThread:2043206 [sender.py:send():379] send: telemetry +2025-01-04 23:02:53,225 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,226 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,226 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,227 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,228 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,228 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,228 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,228 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,228 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,229 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,229 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,229 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,230 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,230 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,230 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,230 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,231 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,231 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,231 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,231 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,231 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,232 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:53,232 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:02:53,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval +2025-01-04 23:02:53,456 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/oaurte1e-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:02:53,528 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/wemsyofa-media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:02:53,611 INFO wandb-upload_1:2043206 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvdm_1l96 +2025-01-04 23:02:53,693 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:02:53,996 INFO SenderThread:2043206 [sender.py:send_request_log_artifact():1525] logged artifact run-cxddxu7z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxNDM4Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjUyMA==', 'latestArtifact': None}} +2025-01-04 23:02:53,996 DEBUG SenderThread:2043206 [sender.py:send():379] send: files +2025-01-04 23:02:53,997 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json with policy now +2025-01-04 23:02:53,997 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:02:53,997 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:02:53,998 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:54,049 DEBUG SenderThread:2043206 [sender.py:send():379] send: exit +2025-01-04 23:02:54,049 INFO SenderThread:2043206 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:send_exit():588] handling runtime: 320 +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:send_exit():594] send defer +2025-01-04 23:02:54,050 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,050 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 23:02:54,050 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 23:02:54,050 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 1 +2025-01-04 23:02:54,051 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 23:02:54,051 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,051 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 23:02:54,051 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 2 +2025-01-04 23:02:54,051 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 23:02:54,051 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 23:02:54,051 DEBUG SystemMonitor:2043206 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 23:02:54,054 DEBUG SystemMonitor:2043206 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 23:02:54,052 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 23:02:54,107 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 23:02:54,107 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 23:02:54,107 INFO HandlerThread:2043206 [interfaces.py:finish():200] Joined network monitor +2025-01-04 23:02:54,107 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,107 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 23:02:54,108 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 3 +2025-01-04 23:02:54,108 DEBUG SenderThread:2043206 [sender.py:send():379] send: stats +2025-01-04 23:02:54,108 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,108 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 23:02:54,109 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:02:54,109 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 23:02:54,109 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 4 +2025-01-04 23:02:54,109 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:02:54,109 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 23:02:54,238 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/xe1ryk6g-media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:02:54,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:02:54,307 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:02:56,308 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:03:01,041 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:01,041 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:03:01,041 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 23:03:01,042 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,042 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 23:03:01,042 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 5 +2025-01-04 23:03:01,043 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:01,043 DEBUG SenderThread:2043206 [sender.py:send():379] send: history +2025-01-04 23:03:01,043 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: summary_record +2025-01-04 23:03:01,043 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:01,043 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 23:03:01,044 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:03:01,044 DEBUG SenderThread:2043206 [sender.py:send():379] send: summary +2025-01-04 23:03:01,048 INFO SenderThread:2043206 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 23:03:01,048 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,048 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 23:03:01,048 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 6 +2025-01-04 23:03:01,048 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:01,048 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 23:03:01,049 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,049 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 23:03:01,052 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 23:03:01,052 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:01,178 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 7 +2025-01-04 23:03:01,178 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:01,178 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:01,178 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 23:03:01,179 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:01,179 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 23:03:01,310 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:03:01,310 INFO Thread-12 :2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml +2025-01-04 23:03:02,052 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:02,601 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 8 +2025-01-04 23:03:02,601 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:02,601 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:02,601 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 23:03:02,602 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:02,602 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 23:03:02,602 INFO SenderThread:2043206 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 23:03:02,604 INFO SenderThread:2043206 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 23:03:02,606 INFO SenderThread:2043206 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 23:03:02,621 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 9 +2025-01-04 23:03:02,621 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:02,621 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 23:03:02,621 DEBUG SenderThread:2043206 [sender.py:send():379] send: artifact +2025-01-04 23:03:02,845 INFO SenderThread:2043206 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 23:03:02,845 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:02,845 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 23:03:02,845 INFO SenderThread:2043206 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 23:03:03,053 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:03,311 INFO SenderThread:2043206 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:03:03,311 INFO SenderThread:2043206 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files +2025-01-04 23:03:03,312 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml config.yaml +2025-01-04 23:03:03,312 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json wandb-summary.json +2025-01-04 23:03:03,312 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:03:03,314 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt requirements.txt +2025-01-04 23:03:03,314 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-metadata.json wandb-metadata.json +2025-01-04 23:03:03,315 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log output.log +2025-01-04 23:03:03,315 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/diff.patch diff.patch +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 media/videos/eval/4_eval_4_76479c82d6b3a303d94d.mp4 +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 media/videos/eval/1_eval_1_7e23084f9a22c63629ba.mp4 +2025-01-04 23:03:03,316 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 media/videos/eval/2_eval_2_db9e650e70a5bc836144.mp4 +2025-01-04 23:03:03,318 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 media/videos/eval/0_eval_0_c2dd8b2025d43ec844b6.mp4 +2025-01-04 23:03:03,318 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 media/videos/eval/3_eval_3_c7d4589e6ff6e133017f.mp4 +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json media/table/eval/total_success_rate_ci_6_90f757a4d783c43f5fec.table.json +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 10 +2025-01-04 23:03:03,321 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:03,321 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,321 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 23:03:03,321 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 23:03:03,321 INFO SenderThread:2043206 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 23:03:03,503 INFO wandb-upload_1:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/config.yaml +2025-01-04 23:03:03,526 INFO wandb-upload_2:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/wandb-summary.json +2025-01-04 23:03:03,537 INFO wandb-upload_0:2043206 [upload_job.py:push():130] Uploaded file /tmp/tmpgekgoqtewandb/a7z1y1w1-events.out.tfevents.1736049453.leonmkim-ROG-Strix-G15CS-G15CS.2043031.0 +2025-01-04 23:03:03,562 INFO wandb-upload_3:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/requirements.txt +2025-01-04 23:03:03,565 INFO wandb-upload_4:2043206 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/files/output.log +2025-01-04 23:03:03,766 INFO Thread-11 (_thread_body):2043206 [sender.py:transition_state():614] send defer: 11 +2025-01-04 23:03:03,766 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,766 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 23:03:03,766 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,766 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 23:03:03,766 INFO SenderThread:2043206 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 23:03:03,767 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 12 +2025-01-04 23:03:03,768 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,768 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 23:03:03,768 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,768 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 23:03:03,768 INFO SenderThread:2043206 [file_stream.py:finish():601] file stream finish called +2025-01-04 23:03:03,953 INFO SenderThread:2043206 [file_stream.py:finish():605] file stream finish is done +2025-01-04 23:03:03,953 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 13 +2025-01-04 23:03:03,953 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,954 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,954 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 23:03:03,954 INFO SenderThread:2043206 [sender.py:transition_state():614] send defer: 14 +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send():379] send: final +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send():379] send: footer +2025-01-04 23:03:03,954 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: defer +2025-01-04 23:03:03,954 INFO HandlerThread:2043206 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 23:03:03,954 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: defer +2025-01-04 23:03:03,955 INFO SenderThread:2043206 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 23:03:03,955 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:03,955 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:03,955 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 23:03:03,956 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 23:03:03,956 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 23:03:03,956 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: server_info +2025-01-04 23:03:03,958 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 23:03:03,958 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 23:03:04,016 DEBUG SenderThread:2043206 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 23:03:04,016 INFO MainThread:2043206 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 23:03:04,017 INFO MainThread:2043206 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 23:03:04,017 INFO MainThread:2043206 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 23:03:04,017 DEBUG HandlerThread:2043206 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 23:03:04,017 INFO HandlerThread:2043206 [handler.py:finish():882] shutting down handler +2025-01-04 23:03:04,956 INFO WriterThread:2043206 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/run-cxddxu7z.wandb +2025-01-04 23:03:05,016 INFO SenderThread:2043206 [sender.py:finish():1615] shutting down sender +2025-01-04 23:03:05,016 INFO SenderThread:2043206 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 23:03:05,016 INFO SenderThread:2043206 [file_pusher.py:join():175] waiting for file pusher diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug.log b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..16035a0311b46fac1afe21bb6520fc9f8f2c6e08 --- /dev/null +++ b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Configure stats pid to 2043031 +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/settings +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:57:32,880 INFO MainThread:2043031 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug.log +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/logs/debug-internal.log +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z'} +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():619] starting backend +2025-01-04 22:57:32,881 INFO MainThread:2043031 [wandb_init.py:init():623] setting up manager +2025-01-04 22:57:32,885 INFO MainThread:2043031 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:57:32,888 INFO MainThread:2043031 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:57:32,900 INFO MainThread:2043031 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:57:32,907 INFO MainThread:2043031 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:57:33,210 INFO MainThread:2043031 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:57:33,252 INFO MainThread:2043031 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:57:33,252 INFO MainThread:2043031 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:57:33,591 INFO MainThread:2043031 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:57:33,591 INFO MainThread:2043031 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:57:33,591 INFO MainThread:2043031 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:57:33,592 INFO MainThread:2043031 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:57:33,594 INFO MainThread:2043031 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:57:33,595 INFO MainThread:2043031 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/cxddxu7z/tb, True +2025-01-04 22:57:39,851 INFO MainThread:2043031 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 23:03:05,018 WARNING MsgRouterThr:2043031 [router.py:message_loop():77] message_loop has been closed diff --git a/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/run-cxddxu7z.wandb b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/run-cxddxu7z.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2c753de03240f62c04b44990168a71ad26b408f0 Binary files /dev/null and b/cxddxu7z/wandb/run-20250104_225732-cxddxu7z/run-cxddxu7z.wandb differ diff --git a/ka36mlrw/.hydra/config.yaml b/ka36mlrw/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/ka36mlrw/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ka36mlrw/.hydra/hydra.yaml b/ka36mlrw/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43db739d01ef01b6ebae02be7276bde1d53c4620 --- /dev/null +++ b/ka36mlrw/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ka36mlrw + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ka36mlrw/.hydra/overrides.yaml b/ka36mlrw/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ka36mlrw/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ka36mlrw/eval_policy.log b/ka36mlrw/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..a010b984a2c61bce80b1b9a4cce24a94a79d2db0 --- /dev/null +++ b/ka36mlrw/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 22:51:59,611][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:51:59,614][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 22:52:03,602][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/l2vooc6k/episode_rosbags/episode_0_2025-01-04-21-04-00.bag b/l2vooc6k/episode_rosbags/episode_0_2025-01-04-21-04-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..471f96dbb7be157bf8b4265cf6ad176a1a967251 --- /dev/null +++ b/l2vooc6k/episode_rosbags/episode_0_2025-01-04-21-04-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b025d1f7816ca18d7a00997e56b3ee29bcf490728181eb4e4d0f71d2ed4ee6c3 +size 1409375429 diff --git a/l2vooc6k/episode_rosbags/episode_1_2025-01-04-21-04-42.bag b/l2vooc6k/episode_rosbags/episode_1_2025-01-04-21-04-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..7f0c651dd9500768e0604d2d4801670a9616af55 --- /dev/null +++ b/l2vooc6k/episode_rosbags/episode_1_2025-01-04-21-04-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4c372a18c196f8b330adcf224ed7c08b749ac8d9e944a149d6384e5ba5729fd +size 1411971323 diff --git a/l2vooc6k/episode_rosbags/episode_2_2025-01-04-21-05-23.bag b/l2vooc6k/episode_rosbags/episode_2_2025-01-04-21-05-23.bag new file mode 100644 index 0000000000000000000000000000000000000000..024445992824723412227c0967e430fd8fe3e572 --- /dev/null +++ b/l2vooc6k/episode_rosbags/episode_2_2025-01-04-21-05-23.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f746c25921953716aca83431479498143fa71401496086dcaaf7580506dcc70 +size 1412843384 diff --git a/l2vooc6k/episode_rosbags/episode_3_2025-01-04-21-06-04.bag b/l2vooc6k/episode_rosbags/episode_3_2025-01-04-21-06-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..94339e953b2fe5c5e2aff4dce74c93333f1bc95f --- /dev/null +++ b/l2vooc6k/episode_rosbags/episode_3_2025-01-04-21-06-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:256a4618f7282072147f6cdd620542a2088d32ca91b8c41f2be51e5f89ea40f1 +size 1409480738 diff --git a/l2vooc6k/episode_rosbags/episode_4_2025-01-04-21-06-53.bag b/l2vooc6k/episode_rosbags/episode_4_2025-01-04-21-06-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..cb61a8d35cd785e25aa0a5530b77ece8112c3de2 --- /dev/null +++ b/l2vooc6k/episode_rosbags/episode_4_2025-01-04-21-06-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b26c90d72588729f86046889a05836e34065777e23067218c1ea8f15988f8e9a +size 1409152793 diff --git a/nhx7r1z0/episode_rosbags/aligned_depth_to_color_K.npy b/nhx7r1z0/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nhx7r1z0/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nhx7r1z0/episode_rosbags/cam_tf_world.npy b/nhx7r1z0/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nhx7r1z0/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nhx7r1z0/episode_rosbags/color_K.npy b/nhx7r1z0/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nhx7r1z0/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nhx7r1z0/episode_rosbags/depth_K.npy b/nhx7r1z0/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nhx7r1z0/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nhx7r1z0/eval_robot.log b/nhx7r1z0/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..9cec4d359bc04553e80c91d0840cfeb2281efc6e --- /dev/null +++ b/nhx7r1z0/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:28:12,749][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:28:12,752][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/nhx7r1z0/wandb/debug-internal.log b/nhx7r1z0/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..df43b1c3630bdd874b61feff00962755c9ed3bc4 --- /dev/null +++ b/nhx7r1z0/wandb/debug-internal.log @@ -0,0 +1,869 @@ +2025-01-03 22:28:18,557 INFO StreamThr :1983623 [internal.py:wandb_internal():85] W&B internal server running at pid: 1983623, started at: 2025-01-03 22:28:18.556934 +2025-01-03 22:28:18,559 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:28:18,571 INFO WriterThread:1983623 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/run-nhx7r1z0.wandb +2025-01-03 22:28:18,571 DEBUG SenderThread:1983623 [sender.py:send():379] send: header +2025-01-03 22:28:18,586 DEBUG SenderThread:1983623 [sender.py:send():379] send: run +2025-01-03 22:28:18,805 INFO SenderThread:1983623 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files +2025-01-03 22:28:18,805 INFO SenderThread:1983623 [sender.py:_start_run_threads():1188] run started: nhx7r1z0 with start time 1735961298.557627 +2025-01-03 22:28:18,829 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:28:18,830 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:28:18,906 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:28:18,911 DEBUG HandlerThread:1983623 [system_info.py:__init__():26] System info init +2025-01-03 22:28:18,911 DEBUG HandlerThread:1983623 [system_info.py:__init__():41] System info init done +2025-01-03 22:28:18,911 INFO HandlerThread:1983623 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:28:18,911 INFO SystemMonitor:1983623 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:28:18,911 INFO HandlerThread:1983623 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:28:18,912 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:28:18,912 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:28:18,913 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:28:18,913 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:28:18,914 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:28:18,924 DEBUG HandlerThread:1983623 [system_info.py:probe():152] Probing system +2025-01-03 22:28:18,925 DEBUG HandlerThread:1983623 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_info.py:probe():200] Probing system done +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:28:18.924324', 'startedAt': '2025-01-04T03:28:18.544196', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.393, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.982, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.924, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 844.8166198730469}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:28:18,930 INFO HandlerThread:1983623 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:28:18,930 INFO HandlerThread:1983623 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_info.py:_save_code():44] Saving code +2025-01-03 22:28:18,932 DEBUG HandlerThread:1983623 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:28:18,932 DEBUG HandlerThread:1983623 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:28:18,993 DEBUG HandlerThread:1983623 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:28:18,993 INFO HandlerThread:1983623 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:28:18,997 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:28:18,997 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:28:18,997 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:28:18,997 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:28:19,229 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:28:19,229 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:28:19,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:28:19,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:28:19,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:19,232 INFO HandlerThread:1983623 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/tb" + save: true +} +control { + relay_id: "513ce0d58ed945ebb8be16b792d16108" +} +_info { + stream_id: "nhx7r1z0" +} + +2025-01-03 22:28:19,232 DEBUG HandlerThread:1983623 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:28:19,244 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/uah3b2wt-wandb-metadata.json +2025-01-03 22:28:19,278 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/nsdn662v-diff.patch +2025-01-03 22:28:19,329 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/4a6tmd9s-code/FISH/eval_robot.py +2025-01-03 22:28:19,353 DEBUG SenderThread:1983623 [sender.py:send():379] send: telemetry +2025-01-03 22:28:19,353 DEBUG SenderThread:1983623 [sender.py:send():379] send: tbrecord +2025-01-03 22:28:19,353 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:28:19,353 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 with policy live +2025-01-03 22:28:19,543 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/56emz1jk-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:28:19,806 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH/eval_robot.py +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/diff.patch +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-metadata.json +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH +2025-01-03 22:28:20,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:21,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:21,806 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:22,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:23,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:24,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:24,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:24,995 DEBUG SenderThread:1983623 [sender.py:send():379] send: config +2025-01-03 22:28:25,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:26,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:27,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:27,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:28,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:29,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:29,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:29,808 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:30,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:31,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:31,809 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:32,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:33,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:33,809 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:34,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:28:34,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:28:34,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:34,384 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:35,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:36,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:37,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:38,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:39,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:40,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:40,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:41,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:42,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:43,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:44,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:45,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:46,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:46,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:47,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:48,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:49,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:28:49,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:28:49,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:49,814 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:50,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:51,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:51,338 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:51,814 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml +2025-01-03 22:28:51,814 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:52,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:53,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:53,815 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:54,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:55,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:56,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:57,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:57,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:58,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:59,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:00,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:01,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:02,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:03,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:03,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:04,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:04,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:04,277 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:05,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:06,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:07,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:08,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:08,374 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:09,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:09,819 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:10,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:11,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:12,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:13,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:14,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:14,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:15,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:16,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:17,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:18,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:18,914 DEBUG SystemMonitor:1983623 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:29:18,915 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:29:19,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:19,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:19,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:19,380 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:19,635 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:29:19,635 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:29:19,635 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 with policy now +2025-01-03 22:29:19,636 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:29:19,637 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:29:19,637 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:29:19,822 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos +2025-01-03 22:29:19,923 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/ux2n87nt-media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:29:20,197 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/y8opkeng-media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:29:20,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:21,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:21,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:22,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:23,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:24,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:25,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:25,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:25,824 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:26,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:27,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:27,825 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:28,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:29,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:30,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:31,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:31,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:32,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:33,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:34,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:34,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:34,277 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:35,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:36,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:36,364 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:37,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:38,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:39,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:40,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:41,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:42,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:42,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:43,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:44,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:45,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:46,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:47,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:47,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:48,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:48,915 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:29:49,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:49,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:49,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:50,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:51,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:52,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:52,382 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:53,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:54,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:55,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:56,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:57,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:58,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:58,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:59,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:00,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:01,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:02,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:03,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:03,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:04,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:04,231 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:04,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:05,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:06,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:07,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:08,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:08,313 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:09,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:10,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:11,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:12,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:13,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:14,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:14,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:15,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:16,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:17,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:17,843 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:30:18,087 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/keyst1n2-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:30:18,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:18,916 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:30:19,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:19,231 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:19,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:19,347 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:19,844 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:20,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:21,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:22,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:23,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:23,845 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:24,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:25,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:25,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:26,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:27,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:28,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:29,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:30,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:31,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:31,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:32,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:33,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:34,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:34,231 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:34,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:35,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:36,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:36,388 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:37,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:38,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:39,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:40,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:41,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:42,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:42,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:43,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:44,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:45,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:46,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:47,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:48,031 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:48,253 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:48,916 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:30:49,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:49,233 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:49,254 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:49,688 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:30:49,688 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 with policy now +2025-01-03 22:30:49,689 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:30:49,689 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:30:49,690 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:30:49,693 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:30:49,981 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/nu95xmxi-media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:30:50,241 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/q85jsv4e-media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:30:50,255 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:51,254 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:51,857 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:52,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:53,255 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:53,315 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:54,258 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:55,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:55,858 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:56,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:57,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:57,859 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:58,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:59,133 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:59,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:00,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:00,695 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:31:00,695 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:31:00,696 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:31:00,860 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:31:00,860 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:31:01,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:01,860 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:02,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:03,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:03,861 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:04,209 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:04,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:04,249 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:04,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:05,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:05,862 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:06,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:07,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:08,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:09,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:09,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:10,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:11,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:12,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:13,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:13,864 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:14,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:14,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:15,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:16,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:17,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:18,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:18,916 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:31:19,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:19,233 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:19,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:19,380 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:20,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:21,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:22,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:23,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:24,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:25,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:25,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:26,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:27,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:28,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:29,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:30,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:30,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:31,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:32,268 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:32,394 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:31:32,394 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 with policy now +2025-01-03 22:31:32,396 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:31:32,397 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:31:32,397 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:31:32,397 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:31:32,648 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/bvxodu4k-media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:31:32,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:31:32,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:31:32,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:31:33,145 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/4wlgxdx7-media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:31:33,270 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:33,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:34,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:34,240 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:34,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:35,272 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:35,327 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:36,275 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:37,278 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:37,872 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:38,277 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:39,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:39,873 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:40,276 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:41,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:41,279 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:42,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:43,280 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:44,280 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:45,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:46,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:46,284 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:47,282 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:48,288 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:48,917 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:31:49,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:49,237 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:49,287 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:50,291 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:51,286 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:51,338 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:52,287 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:53,289 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:54,292 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:55,291 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:56,288 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:57,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:57,391 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:58,387 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:59,388 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:00,389 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:01,388 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:02,250 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:02,390 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:03,390 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:04,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:04,238 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:04,393 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:05,393 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:05,561 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:32:05,561 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:32:05,561 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:32:05,881 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:32:05,882 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:32:06,116 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/9e0wntl7-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:32:06,394 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:07,390 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:07,394 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:07,882 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:08,404 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:09,404 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:09,883 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:10,404 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:11,405 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:11,884 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:12,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:13,331 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:13,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:14,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:15,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:16,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:17,410 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:18,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:18,917 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:32:18,918 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:19,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:19,236 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:19,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:20,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:21,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:22,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:23,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:24,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:24,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:25,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:26,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:27,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:28,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:29,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:29,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:30,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:31,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:31,891 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:32,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:33,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:34,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:34,239 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:34,374 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:34,425 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:34,977 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:32:34,977 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 with policy now +2025-01-03 22:32:34,978 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:32:34,978 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:32:34,978 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:32:34,979 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:32:35,229 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/y1m9gucn-media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:32:35,425 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:32:36,174 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/wazreo3l-media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:32:36,426 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:37,427 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:38,430 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:39,433 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:39,893 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:40,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:40,437 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:41,424 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:42,433 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:43,434 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:43,895 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:44,428 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:45,438 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:46,251 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:46,436 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:47,435 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:48,435 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:48,918 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:32:49,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:49,244 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:49,431 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:50,437 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:51,298 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:51,437 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:52,433 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:53,440 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:53,619 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:32:53,620 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:32:53,621 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:32:53,898 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:32:53,898 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:32:54,465 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:55,443 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:55,899 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:56,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:56,466 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:57,450 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:58,449 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:59,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:59,900 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:00,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:01,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:02,253 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:02,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:03,473 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:04,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:04,237 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:04,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:05,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:05,902 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:06,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:07,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:07,505 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:08,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:09,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:10,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:11,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:12,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:13,305 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:13,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:13,905 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:14,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:15,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:16,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:17,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:18,439 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:18,481 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:18,918 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:33:19,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:19,238 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:19,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:19,907 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:20,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:21,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:21,908 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:22,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:23,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:24,253 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:24,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:25,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:26,454 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:27,096 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:33:27,096 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 with policy now +2025-01-03 22:33:27,100 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:33:27,106 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:27,106 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:27,106 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:27,358 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/xzlexb6d-media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:33:27,454 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:27,910 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:27,911 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:33:27,911 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:33:27,911 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:33:28,188 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/66kwga2b-media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:33:28,470 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:29,257 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:29,472 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:30,469 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:31,474 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:31,912 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:32,473 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:33,475 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:34,251 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:34,251 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:34,472 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:34,607 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:35,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:35,913 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:36,487 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:37,482 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:38,485 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:39,486 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:40,268 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:40,487 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:41,491 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:42,495 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:43,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:44,495 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:45,492 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:46,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:46,488 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:47,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:48,488 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:48,919 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:33:49,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:49,248 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:49,495 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:50,494 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:51,506 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:51,607 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:33:51,608 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:33:51,893 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:33:51,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:51,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:33:51,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:33:51,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media +2025-01-03 22:33:51,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval +2025-01-03 22:33:51,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table +2025-01-03 22:33:52,094 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/yycw34ng-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:33:52,122 INFO wandb-upload_1:1983623 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp99ywx_oj +2025-01-03 22:33:52,184 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/3klijqxb-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:33:52,282 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:33:52,291 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:33:52,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:52,523 INFO SenderThread:1983623 [sender.py:send_request_log_artifact():1525] logged artifact run-nhx7r1z0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3OTgyNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4ODU1OQ==', 'latestArtifact': None}} +2025-01-03 22:33:52,524 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:33:52,524 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json with policy now +2025-01-03 22:33:52,524 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:52,524 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:52,524 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,525 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,525 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:52,525 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,529 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,529 DEBUG SenderThread:1983623 [sender.py:send():379] send: telemetry +2025-01-03 22:33:52,529 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,530 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,530 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,530 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,530 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,530 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,531 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,531 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,531 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,531 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,532 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,532 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,532 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,533 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,533 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,533 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,533 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,534 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,534 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,534 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,534 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,535 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,535 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:33:52,783 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/3jtopk98-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:33:52,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:33:52,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:33:52,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:33:52,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval +2025-01-03 22:33:52,949 INFO wandb-upload_0:1983623 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9j7x7ffg +2025-01-03 22:33:53,104 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/57uppe4r-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:33:53,439 INFO SenderThread:1983623 [sender.py:send_request_log_artifact():1525] logged artifact run-nhx7r1z0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3OTgzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4ODU2MQ==', 'latestArtifact': None}} +2025-01-03 22:33:53,439 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:33:53,440 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json with policy now +2025-01-03 22:33:53,440 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:53,440 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:53,441 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:53,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:53,668 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/nyqb6hqz-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:33:53,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:53,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:33:54,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:55,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:56,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:57,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:58,259 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:58,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:59,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:00,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:01,101 DEBUG SenderThread:1983623 [sender.py:send():379] send: exit +2025-01-03 22:34:01,101 INFO SenderThread:1983623 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:send_exit():588] handling runtime: 342 +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:send_exit():594] send defer +2025-01-03 22:34:01,102 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,102 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:34:01,102 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:34:01,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,103 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:34:01,103 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,103 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:34:01,103 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:34:01,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,103 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:34:01,103 INFO HandlerThread:1983623 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:34:01,103 DEBUG SystemMonitor:1983623 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:34:01,103 DEBUG SystemMonitor:1983623 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:34:01,104 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:34:01,104 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:34:01,155 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:34:01,156 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:34:01,156 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:34:01,156 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,156 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:34:01,156 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:34:01,156 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:34:01,156 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,156 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:34:01,157 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,157 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:34:01,157 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:34:01,157 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,157 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:34:01,923 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:34:01,923 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:34:07,698 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:07,698 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:07,698 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:34:07,699 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,699 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:34:07,699 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:34:07,699 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:07,699 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:34:07,700 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:34:07,700 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:07,700 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:34:07,700 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:34:07,701 DEBUG SenderThread:1983623 [sender.py:send():379] send: summary +2025-01-03 22:34:07,704 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:34:07,705 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,705 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:34:07,705 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:34:07,705 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:07,705 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:34:07,705 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,705 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:34:07,708 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:07,854 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:34:07,854 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:07,854 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:34:07,854 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,854 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:34:07,925 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml +2025-01-03 22:34:07,926 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:34:08,102 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:09,780 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:34:09,780 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:09,780 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:09,780 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:34:09,780 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:09,781 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:34:09,781 INFO SenderThread:1983623 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:34:09,783 INFO SenderThread:1983623 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:34:09,785 INFO SenderThread:1983623 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:34:09,800 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:34:09,800 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:09,800 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:34:09,800 DEBUG SenderThread:1983623 [sender.py:send():379] send: artifact +2025-01-03 22:34:09,926 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:34:10,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:10,147 INFO SenderThread:1983623 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:34:10,148 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:10,148 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:34:10,148 INFO SenderThread:1983623 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:34:10,927 INFO SenderThread:1983623 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files +2025-01-03 22:34:10,927 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml config.yaml +2025-01-03 22:34:10,927 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json wandb-summary.json +2025-01-03 22:34:10,928 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:34:10,928 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt requirements.txt +2025-01-03 22:34:10,930 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:34:10,930 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log output.log +2025-01-03 22:34:10,930 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/diff.patch diff.patch +2025-01-03 22:34:10,932 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:34:10,933 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:34:10,933 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:34:10,934 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:34:10,934 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:34:10,935 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:34:10,936 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:34:10,936 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:34:10,936 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:34:10,936 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:10,937 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:10,937 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:34:10,937 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:10,937 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:34:10,937 INFO SenderThread:1983623 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:34:11,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:11,104 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:11,109 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml +2025-01-03 22:34:11,117 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:34:11,157 INFO wandb-upload_3:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt +2025-01-03 22:34:11,162 INFO wandb-upload_4:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:34:11,183 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/x5izr4j8-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:34:11,383 INFO Thread-11 (_thread_body):1983623 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:34:11,383 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,383 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:34:11,384 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,384 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:34:11,384 INFO SenderThread:1983623 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:34:11,385 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:34:11,385 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,385 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:34:11,385 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,385 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:34:11,385 INFO SenderThread:1983623 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:34:11,559 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,559 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:34:11,559 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:34:11,560 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,560 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:34:11,560 DEBUG SenderThread:1983623 [sender.py:send():379] send: final +2025-01-03 22:34:11,560 DEBUG SenderThread:1983623 [sender.py:send():379] send: footer +2025-01-03 22:34:11,560 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,560 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:34:11,561 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:11,561 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:34:11,561 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:11,561 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:11,561 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:34:11,562 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:34:11,564 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:34:11,564 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:11,619 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:11,619 INFO MainThread:1983623 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:34:11,620 INFO MainThread:1983623 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:34:11,620 INFO MainThread:1983623 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:34:11,620 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:34:11,620 INFO HandlerThread:1983623 [handler.py:finish():882] shutting down handler +2025-01-03 22:34:12,561 INFO WriterThread:1983623 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/run-nhx7r1z0.wandb +2025-01-03 22:34:12,619 INFO SenderThread:1983623 [sender.py:finish():1615] shutting down sender +2025-01-03 22:34:12,620 INFO SenderThread:1983623 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:34:12,620 INFO SenderThread:1983623 [file_pusher.py:join():175] waiting for file pusher diff --git a/nhx7r1z0/wandb/debug.log b/nhx7r1z0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..153a0f6023548f0b0ea0986c60a9d27eb643f22a --- /dev/null +++ b/nhx7r1z0/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Configure stats pid to 1983448 +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/settings +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug.log +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug-internal.log +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:28:18,551 INFO MainThread:1983448 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0'} +2025-01-03 22:28:18,551 INFO MainThread:1983448 [wandb_init.py:init():619] starting backend +2025-01-03 22:28:18,551 INFO MainThread:1983448 [wandb_init.py:init():623] setting up manager +2025-01-03 22:28:18,556 INFO MainThread:1983448 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:28:18,557 INFO MainThread:1983448 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:28:18,570 INFO MainThread:1983448 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:28:18,581 INFO MainThread:1983448 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:28:18,829 INFO MainThread:1983448 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:28:18,901 INFO MainThread:1983448 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:28:18,901 INFO MainThread:1983448 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:28:19,232 INFO MainThread:1983448 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:28:19,232 INFO MainThread:1983448 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/tb, True +2025-01-03 22:28:24,995 INFO MainThread:1983448 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:34:12,621 WARNING MsgRouterThr:1983448 [router.py:message_loop():77] message_loop has been closed diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ce490d5de8245a5cf9af43902e16a989a4b915f --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735961298 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/diff.patch b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9bf74e0d903538392815fb5b2f4d10dd914f2d4e --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735961304.996037]: resetting environment +[INFO] [1735961304.998840]: cleared current plan +[INFO] [1735961304.999986]: moving to home +[INFO] [1735961306.002375]: reached home +[INFO] [1735961306.004095]: reset action history +[INFO] [1735961308.713836]: environment reset +Starting episode 0 +[INFO] [1735961308.716458]: resetting environment +[INFO] [1735961308.719444]: cleared current plan +[INFO] [1735961308.720266]: moving to home +[INFO] [1735961309.722679]: reached home +[INFO] [1735961309.724125]: reset action history +[INFO] [1735961312.430165]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735961327.923090]: resetting environment +[INFO] [1735961327.928542]: cleared current plan +[INFO] [1735961327.929624]: moving to home +[INFO] [1735961328.931642]: reached home +[INFO] [1735961328.932945]: reset action history +[INFO] [1735961331.641131]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/episode_rosbags/episode_0_2025-01-03-22-28-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1983448.log +started roslaunch server http://158.130.50.37:38375/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1983723] +process[print_text-2]: started with pid [1983747] +process[rosbag_record-3]: started with pid [1983748] +started bagging! +For topic gripper_width: timestamp difference is -118491309 for nearest: 1735961348661556346 - target: 1735961348780047655 at idx 242 +[INFO] [1735961358.636307]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1735961359.397732]: Stored episode 1. +[INFO] [1735961359.397950]: Saving video... +[INFO] [1735961359.618200]: Video saved! +Starting episode 1 +[INFO] [1735961359.636506]: resetting environment +[INFO] [1735961359.640584]: cleared current plan +[INFO] [1735961359.640980]: moving to home +[INFO] [1735961363.144089]: reached home +[INFO] [1735961363.144918]: reset action history +[INFO] [1735961365.850474]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961417.511863]: resetting environment +[INFO] [1735961417.514902]: cleared current plan +[INFO] [1735961417.515064]: moving to home +[INFO] [1735961418.516899]: reached home +[INFO] [1735961418.517541]: reset action history +[INFO] [1735961421.224170]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/episode_rosbags/episode_1_2025-01-03-22-30-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1983448.log +started roslaunch server http://158.130.50.37:45971/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1983915] +process[print_text-5]: started with pid [1983916] +process[rosbag_record-6]: started with pid [1983917] +started bagging! +For topic gripper_width: timestamp difference is 124330433 for nearest: 1735961448028223427 - target: 1735961447903892994 at idx 249 +[INFO] [1735961448.208200]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735961448.946074]: Stored episode 2. +[INFO] [1735961448.952796]: Saving video... +[INFO] [1735961449.652708]: Video saved! +Starting episode 2 +[INFO] [1735961449.695402]: resetting environment +[INFO] [1735961449.705898]: cleared current plan +[INFO] [1735961449.707922]: moving to home +[WARN] [1735961449.581301]: Plan exhausted +[WARN] [1735961449.613176]: Plan exhausted +[WARN] [1735961449.650959]: Plan exhausted +[WARN] [1735961449.690444]: Plan exhausted +[INFO] [1735961453.314223]: reached home +[INFO] [1735961453.316618]: reset action history +[INFO] [1735961456.088785]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961460.478792]: resetting environment +[INFO] [1735961460.487868]: cleared current plan +[INFO] [1735961460.493902]: moving to home +[INFO] [1735961461.497274]: reached home +[INFO] [1735961461.497905]: reset action history +[INFO] [1735961464.208501]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/episode_rosbags/episode_2_2025-01-03-22-31-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1983448.log +started roslaunch server http://158.130.50.37:40589/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1984089] +process[print_text-8]: started with pid [1984090] +process[rosbag_record-9]: started with pid [1984091] +started bagging! +For topic gripper_width: timestamp difference is 110444813 for nearest: 1735961471961532622 - target: 1735961471851087809 at idx 102 +[INFO] [1735961491.363405]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735961492.117380]: Stored episode 3. +[INFO] [1735961492.118217]: Saving video... +[INFO] [1735961492.350203]: Video saved! +Starting episode 3 +[INFO] [1735961492.399551]: resetting environment +[INFO] [1735961492.414328]: cleared current plan +[INFO] [1735961492.415175]: moving to home +[INFO] [1735961495.818830]: reached home +[INFO] [1735961495.821203]: reset action history +[INFO] [1735961498.593597]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961525.325946]: resetting environment +[INFO] [1735961525.335211]: cleared current plan +[INFO] [1735961525.337740]: moving to home +[INFO] [1735961526.345110]: reached home +[INFO] [1735961526.347191]: reset action history +[INFO] [1735961529.068218]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/episode_rosbags/episode_3_2025-01-03-22-32-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1983448.log +started roslaunch server http://158.130.50.37:42591/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1984240] +process[print_text-11]: started with pid [1984241] +process[rosbag_record-12]: started with pid [1984265] +started bagging! +For topic gripper_width: timestamp difference is 101824365 for nearest: 1735961551061532102 - target: 1735961550959707737 at idx 249 +preempting episode +[INFO] [1735961553.771972]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735961554.533142]: Stored episode 4. +[INFO] [1735961554.541254]: Saving video... +[INFO] [1735961554.930180]: Video saved! +Starting episode 4 +[INFO] [1735961554.985860]: resetting environment +[INFO] [1735961555.014917]: cleared current plan +[INFO] [1735961555.022172]: moving to home +[WARN] [1735961554.984912]: Plan exhausted +[INFO] [1735961558.440044]: reached home +[INFO] [1735961558.441938]: reset action history +[INFO] [1735961561.239115]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961573.400281]: resetting environment +[INFO] [1735961573.408632]: cleared current plan +[INFO] [1735961573.410010]: moving to home +[INFO] [1735961574.417406]: reached home +[INFO] [1735961574.419577]: reset action history +[INFO] [1735961577.191566]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/episode_rosbags/episode_4_2025-01-03-22-32-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1983448.log +started roslaunch server http://158.130.50.37:34277/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1984407] +process[print_text-14]: started with pid [1984408] +process[rosbag_record-15]: started with pid [1984409] +started bagging! +For topic gripper_width: timestamp difference is 100406408 for nearest: 1735961583428173542 - target: 1735961583327767134 at idx 83 +For topic gripper_width: timestamp difference is 101245685 for nearest: 1735961584461528821 - target: 1735961584360283136 at idx 92 +For topic gripper_width: timestamp difference is 107565310 for nearest: 1735961592261527922 - target: 1735961592153962612 at idx 181 +For topic gripper_width: timestamp difference is 112568827 for nearest: 1735961598428222390 - target: 1735961598315653563 at idx 249 +For topic gripper_width: timestamp difference is 114030897 for nearest: 1735961600228231966 - target: 1735961600114201069 at idx 249 +[INFO] [1735961605.819694]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1735961606.598159]: Stored episode 5. +[INFO] [1735961606.605584]: Saving video... +[INFO] [1735961607.062732]: Video saved! +[INFO] [1735961607.100470]: resetting environment +[INFO] [1735961607.114451]: cleared current plan +[INFO] [1735961607.117886]: moving to home +[INFO] [1735961610.624288]: reached home +[INFO] [1735961610.625974]: reset action history +[INFO] [1735961613.469864]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-metadata.json b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..80589461f103f2600565569d0a6c479576ea6e5e --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:28:18.924324", + "startedAt": "2025-01-04T03:28:18.544196", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.393, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.982, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.924, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 844.8166198730469 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..027ed301e53dbbcef3d9e868a22ef6980162ac36 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3b79de18ba2122e2eecafa013d1c65c981cc55bc451baebaa06c8943c65a868f", "size": 1211156, "path": "media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4"}, "global_step": 653, "_timestamp": 1735961632.2067335, "_runtime": 333.64910650253296, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "f5d85a1429df5e5a3e60b72af66dc6f95d17129210f24776fc22b6a8be7ce7db", "size": 1175549, "path": "media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4"}, "eval/num_success": 2.0, "episode": 25.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.09420213103294373, "eval/success_rate_ub_0.01": 0.8653537631034851, "eval/success_rate_lb_0.025": 0.08973243832588196, "eval/success_rate_ub_0.025": 0.8341690897941589, "eval/success_rate_lb_0.05": 0.12645964324474335, "eval/success_rate_ub_0.05": 0.7050612568855286, "eval/success_rate_lb_0.1": 0.11327370256185532, "eval/success_rate_ub_0.1": 0.6491125226020813, "eval/2_eval": {"_type": "video-file", "sha256": "3966ddb8583e8b4d202c0e9183dc73074df0e024050b9367b17669b1818d8767", "size": 1161759, "path": "media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c408001aca8abf0c7ef9304a8f8dbd796c01659db8c6e2de22e55aaf8019fe4a", "size": 1144006, "path": "media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d93a45e1282e6c91c0f9e54b9ae944c5239c7467e2b08b9b2fc2066d7862812c", "size": 1230887, "path": "media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "152e8da8630452f2ed70d80bc2edc298c7b89d62a8ccf7c81614e14ed3360e9c", "size": 402, "artifact_path": "wandb-client-artifact://uhj9z5nvudb3xzpw0yqbvmd21yoz8t8yfdkbsfztmukla9ba30679kty1e25vuzirixmrywyi6tv0ykwu4274x5dy76eg9wgrgaer2d5wkdqt6sxhg9zjf4yn6ujbtk0/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://g39rppyybxr2trvb88n9wkjzkbv5k9ow7m58ld1bb5dgn9q2kyrcg84e1z85352nj690yyr671visec5r84gvpmojllx1no0qltcdn9cnaw3n5r8fg1imui1b7l4a53f:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 22, "total_num_episodes": 25, "total_success_rate": 0.88, "total_success_rate_lb_0.01": 0.7029817103933635, "total_success_rate_ub_0.01": 0.9734671934492175, "total_success_rate_lb_0.025": 0.6976236402593188, "total_success_rate_ub_0.025": 0.9621387421160531, "total_success_rate_lb_0.05": 0.7217742651257426, "total_success_rate_ub_0.05": 0.9598798117611944, "total_success_rate_lb_0.1": 0.7949399984783139, "total_success_rate_ub_0.1": 0.9499553627365946, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "5b1710e6ff2307150c3cc2fc63ccf665b8ee7925515cee52dc2c7b87522ba422", "size": 506, "artifact_path": "wandb-client-artifact://lci3bgqptghhgi1zidtratroqte9ueoy4jci0cipz19u1kuqcjj0gquytpyf7d4ejkzoj38v8jxp4uxsiot01tn2c6846szphv5y2gfz6ul9ty708q4icq411iw7cd2s/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://y2r3xjxmjl24loojq46q8r0yeydxhyxrujv9lwlve0nih2ezaeifxhs4zilrmy0hw6qfupbvplwso7t0ffc5b3fqbhux56xd2vcla626gn7ewy3coj32m3zwhho2vo58:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 342}, "eval/total_success_rate": 0.8799999952316284, "eval/total_success_rate_lb_0.01": 0.70298171043396, "eval/total_success_rate_ub_0.01": 0.9734671711921692, "eval/total_success_rate_lb_0.025": 0.697623610496521, "eval/total_success_rate_ub_0.025": 0.9621387124061584, "eval/total_success_rate_lb_0.05": 0.7217742800712585, "eval/total_success_rate_ub_0.05": 0.9598798155784607, "eval/total_success_rate_lb_0.1": 0.7949399948120117, "eval/total_success_rate_ub_0.1": 0.9499553442001343} \ No newline at end of file diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/run-nhx7r1z0.wandb b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/run-nhx7r1z0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1ce692f3aee21269cf565d93c5300558a69f04a5 Binary files /dev/null and b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/run-nhx7r1z0.wandb differ diff --git a/snapshot_14249.pt b/snapshot_14249.pt new file mode 100644 index 0000000000000000000000000000000000000000..700d05ecb0f6f1cd95dfe215c8b3794b1bdf6ca7 --- /dev/null +++ b/snapshot_14249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5695588aa94af91375231c79e9bdcf8ed2c6ecd0cde39615876a0e2fc8b166f +size 910102934 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b20e6bd8381d61a9d12a5755621674850725c715 --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77115742cdd4f0ce34ad00014142dc2894b7fc58758c20a26c37d2060192da56 +size 910102424 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f756dec04c06779bb20eb53042b89d1621f5dea --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8affdd645b9795c1c0fe6dec6b9665caa87c89b40b4c8226eb97220135b5aa6 +size 910102424 diff --git a/ukc5qfsm/episode_rosbags/episode_0_2025-01-03-22-16-52.bag b/ukc5qfsm/episode_rosbags/episode_0_2025-01-03-22-16-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..d127fab83e3cefacd7d3088b2d18801101ec8c13 --- /dev/null +++ b/ukc5qfsm/episode_rosbags/episode_0_2025-01-03-22-16-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f8d76d3897e7d9b757f8c2d6c235f07d5f46423d7efa99cb805c7616d1dd93 +size 1409960277 diff --git a/ukc5qfsm/episode_rosbags/episode_1_2025-01-03-22-17-31.bag b/ukc5qfsm/episode_rosbags/episode_1_2025-01-03-22-17-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..4becfa8bc053bb1d08f5257497cdb583d47415f2 --- /dev/null +++ b/ukc5qfsm/episode_rosbags/episode_1_2025-01-03-22-17-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a7909837f543445823a60a594410b4566be955f03659f0aac4f07df78b4ef85 +size 1449749289 diff --git a/ukc5qfsm/episode_rosbags/episode_2_2025-01-03-22-18-35.bag b/ukc5qfsm/episode_rosbags/episode_2_2025-01-03-22-18-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..39e0a517a62e43078c579695e0d56fd47e3670f6 --- /dev/null +++ b/ukc5qfsm/episode_rosbags/episode_2_2025-01-03-22-18-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e2706ad5be7d53e04ffba79f79f9230a61d1efdb78324ebfb329081a1d9980d +size 1454809286 diff --git a/ukc5qfsm/episode_rosbags/episode_3_2025-01-03-22-19-19.bag b/ukc5qfsm/episode_rosbags/episode_3_2025-01-03-22-19-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..61d07928a193072605867e3d639e7cca68d23997 --- /dev/null +++ b/ukc5qfsm/episode_rosbags/episode_3_2025-01-03-22-19-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b496bb16b33ba9c5c18a8f108c3339045fc76401a0f236595305341d0d5536e2 +size 1412629371 diff --git a/ukc5qfsm/episode_rosbags/episode_4_2025-01-03-22-20-02.bag b/ukc5qfsm/episode_rosbags/episode_4_2025-01-03-22-20-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..c1ac888eb59af556cd1532a932f2f3b695d24a2c --- /dev/null +++ b/ukc5qfsm/episode_rosbags/episode_4_2025-01-03-22-20-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e427ec8dcbebb89b921a2b4e4f8221452081e92a1bc09c4b99b69a44b6570407 +size 1454701133 diff --git a/xt2ne5el/.hydra/config.yaml b/xt2ne5el/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/xt2ne5el/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/xt2ne5el/.hydra/hydra.yaml b/xt2ne5el/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b67d5b98d0502017fb500c6843f2a370caf15d9 --- /dev/null +++ b/xt2ne5el/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/xt2ne5el/.hydra/overrides.yaml b/xt2ne5el/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/xt2ne5el/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/xt2ne5el/episode_rosbags/aligned_depth_to_color_K.npy b/xt2ne5el/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/xt2ne5el/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/xt2ne5el/episode_rosbags/cam_tf_world.npy b/xt2ne5el/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/xt2ne5el/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/xt2ne5el/episode_rosbags/color_K.npy b/xt2ne5el/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/xt2ne5el/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/xt2ne5el/episode_rosbags/depth_K.npy b/xt2ne5el/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/xt2ne5el/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/xt2ne5el/eval_robot.log b/xt2ne5el/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..58f21ef2224239e693858c546a22cb358c65cf47 --- /dev/null +++ b/xt2ne5el/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:52:03,548][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:52:03,551][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/xt2ne5el/eval_video/0_eval.mp4 b/xt2ne5el/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d791024a85bb544ad506f20297bf7f0c1505435a --- /dev/null +++ b/xt2ne5el/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa08c14e432d2c4b05aceaa9f0977944236cabd145c3b9c170f6360cc84f9432 +size 1189596 diff --git a/xt2ne5el/eval_video/1_eval.mp4 b/xt2ne5el/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac4cf6b0690ccc5b71b6833d6a90d09f13b6ec37 --- /dev/null +++ b/xt2ne5el/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:537b6b4a557eb4c5962a21f55dda5f3811e7108c6079cc0eb3048c5af0d540ad +size 1146723 diff --git a/xt2ne5el/eval_video/2_eval.mp4 b/xt2ne5el/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f6afa8d6f1df162d5f03d3179379e2277d4bf4fe --- /dev/null +++ b/xt2ne5el/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6556ec2efb80937199535739c4e27a9beaddb7b2ad50916a814d28d7448b26ef +size 1185383 diff --git a/xt2ne5el/eval_video/3_eval.mp4 b/xt2ne5el/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c894dc3f3f51269a622a93ac0bc59548c8621391 --- /dev/null +++ b/xt2ne5el/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55177e1b719e0936946ef43be81658c892606b99c2d7655ffae875d7f5e7ccf4 +size 1156491 diff --git a/xt2ne5el/eval_video/4_eval.mp4 b/xt2ne5el/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a87f11dd95bb4e333bb489c9b26fff6529db5eb5 --- /dev/null +++ b/xt2ne5el/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf759c4daff03b0779d3b0eb31132515fc0ec00ab4a5a97f0e269cc79c129ee1 +size 1213476 diff --git a/xt2ne5el/tb/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 b/xt2ne5el/tb/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 new file mode 100644 index 0000000000000000000000000000000000000000..c43ce3b5607f4b9965fee63264dc0811b4d375b4 --- /dev/null +++ b/xt2ne5el/tb/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e834ec7ec912b9668db18cd6dcf21ee1e3624143032ceba89337f31d2e3a1687 +size 6539 diff --git a/xt2ne5el/wandb/debug-internal.log b/xt2ne5el/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ee264f72d3b7b6386cd735accb031f07872a53f3 --- /dev/null +++ b/xt2ne5el/wandb/debug-internal.log @@ -0,0 +1,809 @@ +2025-01-04 22:52:09,336 INFO StreamThr :2042123 [internal.py:wandb_internal():85] W&B internal server running at pid: 2042123, started at: 2025-01-04 22:52:09.336087 +2025-01-04 22:52:09,337 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:52:09,350 INFO WriterThread:2042123 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/run-xt2ne5el.wandb +2025-01-04 22:52:09,351 DEBUG SenderThread:2042123 [sender.py:send():379] send: header +2025-01-04 22:52:09,358 DEBUG SenderThread:2042123 [sender.py:send():379] send: run +2025-01-04 22:52:09,566 INFO SenderThread:2042123 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files +2025-01-04 22:52:09,566 INFO SenderThread:2042123 [sender.py:_start_run_threads():1188] run started: xt2ne5el with start time 1736049129.336554 +2025-01-04 22:52:09,581 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:52:09,581 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:52:09,628 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:52:09,633 DEBUG HandlerThread:2042123 [system_info.py:__init__():26] System info init +2025-01-04 22:52:09,633 DEBUG HandlerThread:2042123 [system_info.py:__init__():41] System info init done +2025-01-04 22:52:09,633 INFO HandlerThread:2042123 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:52:09,634 INFO SystemMonitor:2042123 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:52:09,634 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:52:09,634 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:52:09,635 INFO HandlerThread:2042123 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:52:09,635 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:52:09,636 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:52:09,636 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:52:09,647 DEBUG HandlerThread:2042123 [system_info.py:probe():152] Probing system +2025-01-04 22:52:09,649 DEBUG HandlerThread:2042123 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_info.py:probe():200] Probing system done +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:52:09.647525', 'startedAt': '2025-01-05T03:52:09.324843', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.7805000000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4498.264, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.007, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 787.8546829223633}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:52:09,654 INFO HandlerThread:2042123 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:52:09,654 INFO HandlerThread:2042123 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_info.py:_save_code():44] Saving code +2025-01-04 22:52:09,655 DEBUG HandlerThread:2042123 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:52:09,655 DEBUG HandlerThread:2042123 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:52:09,718 DEBUG HandlerThread:2042123 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:52:09,718 INFO HandlerThread:2042123 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:52:09,721 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:52:09,721 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:52:09,721 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:52:09,722 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:52:09,949 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/v8ig4z7x-wandb-metadata.json +2025-01-04 22:52:09,962 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:52:09,962 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:52:09,964 DEBUG SenderThread:2042123 [sender.py:send():379] send: telemetry +2025-01-04 22:52:10,002 INFO wandb-upload_2:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/dp777x6a-diff.patch +2025-01-04 22:52:10,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:10,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:10,005 INFO HandlerThread:2042123 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/tb" + save: true +} +control { + relay_id: "b91b4588e7fe4d55a0ffb1941cedd315" +} +_info { + stream_id: "xt2ne5el" +} + +2025-01-04 22:52:10,005 DEBUG HandlerThread:2042123 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:52:10,006 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:10,017 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/hwo6bjcq-code/FISH/eval_robot.py +2025-01-04 22:52:10,158 DEBUG SenderThread:2042123 [sender.py:send():379] send: tbrecord +2025-01-04 22:52:10,158 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:52:10,158 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 with policy live +2025-01-04 22:52:10,335 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/38td6c0n-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-metadata.json +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH/eval_robot.py +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/diff.patch +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code +2025-01-04 22:52:10,569 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH +2025-01-04 22:52:10,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:11,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:12,567 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:12,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:13,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:14,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:14,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:15,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:16,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:17,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:18,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:19,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:19,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:20,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:21,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:22,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:23,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:24,962 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:24,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:25,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:25,051 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:25,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:26,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:27,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:28,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:29,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:30,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:30,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:31,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:32,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:33,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:34,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:35,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:35,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:36,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:37,871 DEBUG SenderThread:2042123 [sender.py:send():379] send: config +2025-01-04 22:52:37,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:38,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:39,962 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:39,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:40,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:40,574 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:40,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:41,087 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:41,575 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml +2025-01-04 22:52:41,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:42,575 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:42,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:43,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:44,576 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:44,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:45,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:46,349 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:46,576 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:46,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:47,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:48,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:49,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:50,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:51,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:51,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:52,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:53,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:54,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:54,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:55,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:55,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:56,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:57,057 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:57,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:58,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:59,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:00,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:01,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:02,881 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:02,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:03,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:04,581 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:04,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:05,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:06,582 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:06,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:07,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:08,582 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:08,881 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:08,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:09,637 DEBUG SystemMonitor:2042123 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:53:09,638 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:53:09,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:09,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:09,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:10,972 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:11,972 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:12,583 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:12,973 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:13,973 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:14,461 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:14,584 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:14,973 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:15,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:16,584 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:16,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:17,981 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:18,981 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:19,466 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:19,981 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:20,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:21,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:22,586 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:22,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:23,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:24,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:24,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:25,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:25,071 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:25,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:26,984 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:27,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:28,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:29,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:30,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:30,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:31,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:32,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:33,782 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:53:33,782 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 with policy now +2025-01-04 22:53:33,783 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:53:33,784 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:53:33,784 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:53:33,787 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:53:33,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:34,066 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/cm9c9j2x-media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:53:34,589 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:34,589 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos +2025-01-04 22:53:34,864 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/7rspm7xz-media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:53:34,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:35,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:35,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:36,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:36,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:37,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:38,591 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:38,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:39,638 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:53:39,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:39,964 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:39,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:40,591 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:40,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:41,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:41,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:42,592 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:53:42,853 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wygl29c4-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:53:42,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:43,592 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:53:43,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:44,593 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:44,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:45,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:46,326 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:46,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:47,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:48,594 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:48,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:49,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:50,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:51,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:51,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:52,595 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:52,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:53,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:54,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:54,964 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:55,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:55,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:56,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:57,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:57,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:58,598 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:58,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:59,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:00,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:01,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:02,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:03,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:03,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:04,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:05,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:06,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:07,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:08,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:08,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:09,638 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:54:09,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:09,964 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:10,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:10,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:11,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:12,992 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:13,996 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:14,084 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:14,344 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:54:14,344 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 with policy now +2025-01-04 22:54:14,347 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:54:14,347 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:54:14,347 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:54:14,348 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:54:14,598 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/v9a0sdvp-media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:54:14,894 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/bifo173a-media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:54:14,999 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:15,996 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:16,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:17,001 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:18,001 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:18,604 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:18,998 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:19,813 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:20,008 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:20,604 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:20,997 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:22,008 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:22,605 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:23,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:23,606 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:54:24,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:24,440 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:54:24,441 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:54:24,441 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:54:24,606 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:54:24,606 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:24,974 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:24,976 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:24,976 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:25,017 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:26,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:26,607 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:27,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:28,003 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:28,607 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:29,003 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:30,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:30,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:31,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:32,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:33,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:34,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:35,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:35,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:36,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:37,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:38,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:38,611 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:39,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:39,639 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:54:39,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:39,965 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:40,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:41,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:41,068 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:42,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:43,006 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:44,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:45,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:46,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:46,281 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:46,613 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:47,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:48,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:49,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:50,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:51,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:51,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:52,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:53,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:54,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:54,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:54,972 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:55,012 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:55,257 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:54:55,257 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 with policy now +2025-01-04 22:54:55,258 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:54:55,258 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:54:55,259 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:54:55,261 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:54:55,519 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/ox298yj8-media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:54:55,616 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:54:55,616 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:54:55,617 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:54:55,886 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/e86uaaj7-media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:54:56,013 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:56,617 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:57,018 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:57,277 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:58,021 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:59,016 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:00,015 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:00,618 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:01,018 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:02,018 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:02,619 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:03,027 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:03,261 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:04,019 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:05,034 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:06,024 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:07,025 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:08,027 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:08,978 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:09,027 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:09,639 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:55:09,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:09,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:10,035 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:11,032 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:12,029 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:13,030 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:14,030 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:14,067 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:15,032 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:16,034 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:16,280 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:55:16,280 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:55:16,280 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:55:16,623 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:55:16,623 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:55:16,861 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wq2r75n7-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:55:17,041 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:18,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:18,624 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:19,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:19,345 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:20,043 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:21,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:22,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:22,625 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:23,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:24,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:24,631 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:24,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:24,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:25,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:26,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:27,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:28,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:29,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:29,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:30,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:31,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:32,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:33,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:34,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:34,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:35,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:36,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:37,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:38,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:39,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:39,640 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:55:39,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:39,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:40,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:40,065 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:41,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:42,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:43,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:44,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:45,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:45,978 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:46,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:47,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:48,045 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:48,428 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:55:48,428 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 with policy now +2025-01-04 22:55:48,430 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:55:48,431 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:55:48,431 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:55:48,431 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:55:48,641 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:55:48,641 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:48,641 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:55:48,642 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:55:48,687 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/lhcbg358-media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:55:48,945 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wf7yoy1l-media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:55:49,047 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:50,049 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:50,642 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:50,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:51,050 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:52,048 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:52,642 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:53,055 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:54,057 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:54,643 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:54,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:54,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:55,053 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:56,059 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:56,121 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:56,644 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:57,069 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:58,067 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:59,067 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:00,066 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:01,071 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:01,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:02,073 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:03,074 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:04,066 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:05,069 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:06,068 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:06,992 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:07,072 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:08,070 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:09,092 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:09,640 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:56:09,975 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:09,975 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:10,087 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:11,086 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:12,089 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:12,116 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:13,092 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:14,091 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:15,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:16,107 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:16,356 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:56:16,357 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:56:16,357 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:56:16,650 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:56:16,651 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:16,651 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:56:16,902 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/605inuye-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:56:17,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:17,861 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:18,107 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:18,651 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:19,100 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:20,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:20,652 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:21,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:22,096 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:22,653 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:22,924 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:23,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:24,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:24,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:24,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:25,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:26,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:27,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:28,055 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:28,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:29,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:30,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:31,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:32,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:33,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:33,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:34,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:35,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:36,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:37,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:38,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:38,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:39,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:39,641 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:56:39,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:39,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:40,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:41,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:42,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:43,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:44,100 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:44,128 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:45,101 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:46,100 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:47,101 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:48,101 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:49,103 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:49,440 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:49,703 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:56:49,703 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 with policy now +2025-01-04 22:56:49,704 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:56:49,704 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:56:49,704 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:56:49,705 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:56:49,963 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/q0wi346c-media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:56:50,102 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:50,662 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:56:50,662 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:50,663 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:56:50,663 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:56:50,956 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/do38pui6-media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:56:51,104 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:52,107 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:53,117 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:54,109 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:54,664 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:54,972 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:54,972 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:55,057 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:55,128 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:56,127 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:57,119 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:58,127 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:58,665 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:59,119 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:00,121 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:00,326 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:57:00,326 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:57:00,577 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:57:00,666 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:00,666 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:00,666 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:00,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media +2025-01-04 22:57:00,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval +2025-01-04 22:57:00,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table +2025-01-04 22:57:00,832 INFO wandb-upload_0:2042123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdjmvylrk +2025-01-04 22:57:00,843 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/j7ylrfmb-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:00,984 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:57:00,992 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:57:01,118 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:01,207 INFO SenderThread:2042123 [sender.py:send_request_log_artifact():1525] logged artifact run-xt2ne5el-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxMDc3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjAwNQ==', 'latestArtifact': None}} +2025-01-04 22:57:01,207 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:57:01,207 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 22:57:01,208 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:01,208 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,208 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:01,209 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,209 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:01,209 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,209 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,210 DEBUG SenderThread:2042123 [sender.py:send():379] send: telemetry +2025-01-04 22:57:01,210 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,210 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,210 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,210 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,211 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,212 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,213 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,213 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,213 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,214 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,214 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,214 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,214 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,215 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,215 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,215 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,215 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,216 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,216 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,216 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,216 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,217 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,217 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:57:01,446 INFO wandb-upload_2:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wgf7axul-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:01,598 INFO wandb-upload_1:2042123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp49l7jzj8 +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval +2025-01-04 22:57:01,970 INFO SenderThread:2042123 [sender.py:send_request_log_artifact():1525] logged artifact run-xt2ne5el-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxMDc4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjAwNg==', 'latestArtifact': None}} +2025-01-04 22:57:01,970 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:57:01,971 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json with policy now +2025-01-04 22:57:01,971 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:01,971 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,971 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:02,118 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:02,142 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/uown77oo-media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:02,386 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/itemdwsk-media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:02,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:02,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:03,118 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:03,273 DEBUG SenderThread:2042123 [sender.py:send():379] send: exit +2025-01-04 22:57:03,273 INFO SenderThread:2042123 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:send_exit():588] handling runtime: 293 +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:send_exit():594] send defer +2025-01-04 22:57:03,274 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,274 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:57:03,274 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:57:03,275 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:57:03,275 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,275 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:57:03,275 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:57:03,275 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:57:03,276 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:57:03,276 DEBUG SystemMonitor:2042123 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:57:03,276 DEBUG SystemMonitor:2042123 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:57:03,327 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:57:03,328 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:57:03,328 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:57:03,328 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,328 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:57:03,328 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:57:03,328 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:57:03,329 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,329 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:57:03,329 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,329 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:57:03,329 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:57:03,329 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,329 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:57:03,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:04,668 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:09,420 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:09,420 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:09,420 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:57:09,421 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,421 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:57:09,421 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:57:09,421 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:09,422 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:09,422 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:09,422 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:09,422 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:57:09,423 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:09,423 DEBUG SenderThread:2042123 [sender.py:send():379] send: summary +2025-01-04 22:57:09,426 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:09,426 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,427 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:57:09,427 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:57:09,427 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:09,427 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:57:09,427 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,427 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:57:09,431 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:09,529 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:57:09,529 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:09,529 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:57:09,530 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,530 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:57:09,670 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:09,670 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml +2025-01-04 22:57:10,274 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:10,472 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:57:10,472 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:10,472 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:10,472 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:57:10,473 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:10,473 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:57:10,473 INFO SenderThread:2042123 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:57:10,475 INFO SenderThread:2042123 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:57:10,477 INFO SenderThread:2042123 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:57:10,489 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:57:10,490 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:10,490 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:57:10,490 DEBUG SenderThread:2042123 [sender.py:send():379] send: artifact +2025-01-04 22:57:10,655 INFO SenderThread:2042123 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:57:10,655 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:10,656 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:57:10,656 INFO SenderThread:2042123 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:57:10,670 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml config.yaml +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json wandb-summary.json +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt requirements.txt +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:57:10,672 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log output.log +2025-01-04 22:57:10,672 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/diff.patch diff.patch +2025-01-04 22:57:10,675 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:57:10,681 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:10,681 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:57:10,681 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:10,681 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:57:10,681 INFO SenderThread:2042123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:57:10,858 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml +2025-01-04 22:57:10,893 INFO wandb-upload_2:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:10,896 INFO wandb-upload_3:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/yx64pkx2-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:10,902 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt +2025-01-04 22:57:10,913 INFO wandb-upload_4:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:11,114 INFO Thread-11 (_thread_body):2042123 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:57:11,114 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,114 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:57:11,114 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,114 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:57:11,114 INFO SenderThread:2042123 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:57:11,117 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:57:11,117 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,117 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:57:11,117 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,117 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:57:11,117 INFO SenderThread:2042123 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:57:11,275 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:11,342 INFO SenderThread:2042123 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:57:11,342 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:57:11,342 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:11,342 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,342 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,343 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:57:11,343 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:57:11,343 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,343 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send():379] send: final +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send():379] send: footer +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,343 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:57:11,344 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:11,344 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:57:11,344 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:11,344 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:57:11,346 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:11,346 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:57:11,346 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:57:11,347 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:11,398 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:11,398 INFO MainThread:2042123 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:57:11,399 INFO MainThread:2042123 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:57:11,399 INFO MainThread:2042123 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:57:11,399 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:57:11,399 INFO HandlerThread:2042123 [handler.py:finish():882] shutting down handler +2025-01-04 22:57:12,346 INFO WriterThread:2042123 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/run-xt2ne5el.wandb +2025-01-04 22:57:12,398 INFO SenderThread:2042123 [sender.py:finish():1615] shutting down sender +2025-01-04 22:57:12,398 INFO SenderThread:2042123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:57:12,398 INFO SenderThread:2042123 [file_pusher.py:join():175] waiting for file pusher diff --git a/xt2ne5el/wandb/debug.log b/xt2ne5el/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ac96bc5b961cedceaf58c28284b68e6c37bd897c --- /dev/null +++ b/xt2ne5el/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Configure stats pid to 2041941 +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/settings +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug.log +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug-internal.log +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el'} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():619] starting backend +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():623] setting up manager +2025-01-04 22:52:09,335 INFO MainThread:2041941 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:52:09,336 INFO MainThread:2041941 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:52:09,348 INFO MainThread:2041941 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:52:09,356 INFO MainThread:2041941 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:52:09,581 INFO MainThread:2041941 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:52:09,623 INFO MainThread:2041941 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:52:09,623 INFO MainThread:2041941 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:52:09,962 INFO MainThread:2041941 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:52:09,963 INFO MainThread:2041941 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:52:09,963 INFO MainThread:2041941 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:52:09,963 INFO MainThread:2041941 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:52:09,965 INFO MainThread:2041941 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:52:09,965 INFO MainThread:2041941 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/tb, True +2025-01-04 22:52:37,870 INFO MainThread:2041941 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:57:12,400 WARNING MsgRouterThr:2041941 [router.py:message_loop():77] message_loop has been closed diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH/eval_robot.py b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..885adcb030b5482ea9db90d6ff52fc49e5653327 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736049129 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/diff.patch b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json new file mode 100644 index 0000000000000000000000000000000000000000..dcaf68ef1835b026b4c1ed74bb951a3735067c55 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[42, 55, 0.7636363636363637, 0.6250906586396852, 0.8734871148316137, 0.6343281566828056, 0.8572508766532698, 0.6523377820544742, 0.8484609349865173, 0.6918580717954774, 0.8336135632622587]]} \ No newline at end of file diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d791024a85bb544ad506f20297bf7f0c1505435a --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa08c14e432d2c4b05aceaa9f0977944236cabd145c3b9c170f6360cc84f9432 +size 1189596 diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac4cf6b0690ccc5b71b6833d6a90d09f13b6ec37 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:537b6b4a557eb4c5962a21f55dda5f3811e7108c6079cc0eb3048c5af0d540ad +size 1146723 diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f6afa8d6f1df162d5f03d3179379e2277d4bf4fe --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6556ec2efb80937199535739c4e27a9beaddb7b2ad50916a814d28d7448b26ef +size 1185383 diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c894dc3f3f51269a622a93ac0bc59548c8621391 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55177e1b719e0936946ef43be81658c892606b99c2d7655ffae875d7f5e7ccf4 +size 1156491 diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a87f11dd95bb4e333bb489c9b26fff6529db5eb5 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf759c4daff03b0779d3b0eb31132515fc0ec00ab4a5a97f0e269cc79c129ee1 +size 1213476 diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0f00675787d060dae09892f187a2be34cd6f53dc --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log @@ -0,0 +1,268 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736049157.871808]: resetting environment +[INFO] [1736049157.876467]: cleared current plan +[INFO] [1736049157.877283]: moving to home +[INFO] [1736049158.880004]: reached home +[INFO] [1736049158.881273]: reset action history +[INFO] [1736049161.589700]: environment reset +Starting episode 0 +[INFO] [1736049161.591455]: resetting environment +[INFO] [1736049161.594285]: cleared current plan +[INFO] [1736049161.595069]: moving to home +[INFO] [1736049162.597270]: reached home +[INFO] [1736049162.599033]: reset action history +[INFO] [1736049165.306609]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736049181.870933]: resetting environment +[INFO] [1736049181.877112]: cleared current plan +[INFO] [1736049181.878894]: moving to home +[INFO] [1736049182.881118]: reached home +[INFO] [1736049182.882370]: reset action history +[INFO] [1736049185.590026]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/episode_rosbags/episode_0_2025-01-04-22-53-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2041941.log +started roslaunch server http://158.130.50.37:41707/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2042251] +process[print_text-2]: started with pid [2042275] +process[rosbag_record-3]: started with pid [2042276] +started bagging! +For topic gripper_width: timestamp difference is 104152985 for nearest: 1736049189818161078 - target: 1736049189714008093 at idx 60 +For topic gripper_width: timestamp difference is 101510286 for nearest: 1736049193418163061 - target: 1736049193316652775 at idx 105 +For topic gripper_width: timestamp difference is 103144354 for nearest: 1736049195418160147 - target: 1736049195315015793 at idx 134 +For topic gripper_width: timestamp difference is 106964563 for nearest: 1736049200218163704 - target: 1736049200111199141 at idx 192 +[INFO] [1736049212.773250]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736049213.569319]: Stored episode 1. +[INFO] [1736049213.569597]: Saving video... +[INFO] [1736049213.768723]: Video saved! +Starting episode 1 +[INFO] [1736049213.784840]: resetting environment +[INFO] [1736049213.794879]: cleared current plan +[INFO] [1736049213.795081]: moving to home +[INFO] [1736049216.897253]: reached home +[INFO] [1736049216.897714]: reset action history +[INFO] [1736049219.605799]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049222.601429]: resetting environment +[INFO] [1736049222.609948]: cleared current plan +[INFO] [1736049222.615810]: moving to home +[INFO] [1736049223.619470]: reached home +[INFO] [1736049223.619912]: reset action history +[INFO] [1736049226.325970]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/episode_rosbags/episode_1_2025-01-04-22-53-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2041941.log +started roslaunch server http://158.130.50.37:34849/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2042386] +process[print_text-5]: started with pid [2042387] +process[rosbag_record-6]: started with pid [2042411] +started bagging! +For topic gripper_width: timestamp difference is -101432269 for nearest: 1736049230751493032 - target: 1736049230852925301 at idx 63 +For topic gripper_width: timestamp difference is 102524062 for nearest: 1736049235918159266 - target: 1736049235815635204 at idx 133 +[INFO] [1736049253.304711]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736049254.076632]: Stored episode 2. +[INFO] [1736049254.085011]: Saving video... +[INFO] [1736049254.299752]: Video saved! +Starting episode 2 +[INFO] [1736049254.350363]: resetting environment +[INFO] [1736049254.362753]: cleared current plan +[INFO] [1736049254.363200]: moving to home +[INFO] [1736049257.769907]: reached home +[INFO] [1736049257.776680]: reset action history +[INFO] [1736049260.557793]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049263.430469]: resetting environment +[INFO] [1736049263.439124]: cleared current plan +[INFO] [1736049263.443056]: moving to home +[INFO] [1736049264.445237]: reached home +[INFO] [1736049264.448941]: reset action history +[INFO] [1736049267.235991]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/episode_rosbags/episode_2_2025-01-04-22-54-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2041941.log +started roslaunch server http://158.130.50.37:34169/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2042522] +process[print_text-8]: started with pid [2042523] +process[rosbag_record-9]: started with pid [2042524] +started bagging! +For topic gripper_width: timestamp difference is 102112274 for nearest: 1736049276818160515 - target: 1736049276716048241 at idx 139 +For topic gripper_width: timestamp difference is 108088932 for nearest: 1736049284218124351 - target: 1736049284110035419 at idx 247 +For topic gripper_width: timestamp difference is 108937871 for nearest: 1736049285218161714 - target: 1736049285109223843 at idx 249 +[INFO] [1736049294.227078]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736049294.996885]: Stored episode 3. +[INFO] [1736049295.002326]: Saving video... +[INFO] [1736049295.215989]: Video saved! +Starting episode 3 +[INFO] [1736049295.263754]: resetting environment +[INFO] [1736049295.276404]: cleared current plan +[INFO] [1736049295.276655]: moving to home +[INFO] [1736049298.385152]: reached home +[INFO] [1736049298.387191]: reset action history +[INFO] [1736049301.218722]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049316.265368]: resetting environment +[INFO] [1736049316.283401]: cleared current plan +[INFO] [1736049316.289140]: moving to home +[INFO] [1736049317.293157]: reached home +[INFO] [1736049317.300600]: reset action history +[INFO] [1736049320.079399]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/episode_rosbags/episode_3_2025-01-04-22-55-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2041941.log +started roslaunch server http://158.130.50.37:36493/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2042692] +process[print_text-11]: started with pid [2042693] +process[rosbag_record-12]: started with pid [2042694] +started bagging! +[INFO] [1736049347.355809]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736049348.158737]: Stored episode 4. +[INFO] [1736049348.160032]: Saving video... +[INFO] [1736049348.380146]: Video saved! +Starting episode 4 +[INFO] [1736049348.438178]: resetting environment +[INFO] [1736049348.458272]: cleared current plan +[INFO] [1736049348.458660]: moving to home +[INFO] [1736049351.867833]: reached home +[INFO] [1736049351.874526]: reset action history +[INFO] [1736049354.671690]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736049375.777769]: resetting environment +[INFO] [1736049375.801215]: cleared current plan +[INFO] [1736049375.805994]: moving to home +[INFO] [1736049376.816532]: reached home +[INFO] [1736049376.825606]: reset action history +[INFO] [1736049379.658668]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/episode_rosbags/episode_4_2025-01-04-22-56-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2041941.log +started roslaunch server http://158.130.50.37:45439/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2042839] +process[print_text-14]: started with pid [2042840] +process[rosbag_record-15]: started with pid [2042841] +started bagging! +[INFO] [1736049408.659974]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736049409.436974]: Stored episode 5. +[INFO] [1736049409.442858]: Saving video... +[INFO] [1736049409.656739]: Video saved! +[INFO] [1736049409.705365]: resetting environment +[INFO] [1736049409.721482]: cleared current plan +[INFO] [1736049409.724526]: moving to home +[INFO] [1736049413.144359]: reached home +[INFO] [1736049413.164234]: reset action history +[INFO] [1736049416.020670]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-metadata.json b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ad3010364f0c61c8ffbf459dfab04096dcb619c1 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:52:09.647525", + "startedAt": "2025-01-05T03:52:09.324843", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.7805000000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4498.264, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.007, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 787.8546829223633 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bcb7d5b613a5d26086cafaa4995a104975ecfcf8 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "fa08c14e432d2c4b05aceaa9f0977944236cabd145c3b9c170f6360cc84f9432", "size": 1189596, "path": "media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4"}, "global_step": 665, "_timestamp": 1736049420.908103, "_runtime": 291.5715489387512, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "537b6b4a557eb4c5962a21f55dda5f3811e7108c6079cc0eb3048c5af0d540ad", "size": 1146723, "path": "media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4"}, "eval/num_success": 0.0, "episode": 55.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "6556ec2efb80937199535739c4e27a9beaddb7b2ad50916a814d28d7448b26ef", "size": 1185383, "path": "media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "55177e1b719e0936946ef43be81658c892606b99c2d7655ffae875d7f5e7ccf4", "size": 1156491, "path": "media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "bf759c4daff03b0779d3b0eb31132515fc0ec00ab4a5a97f0e269cc79c129ee1", "size": 1213476, "path": "media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://zslx27hlyc04urfq1kjhfxgog2u8z54hduvglu61jm7w6nx6w306qnwahca19r3kir7htpuchfe4o21ifc7lwxep8l1yn180l6n2t2g8a9vrzc4zgm4ocdtkri3fno46/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://44gooxi6r2dxyg5fcdnjhle8dfehv69aptjwc0hdv70b76w477o95dp07of4h6vgsidazutmqvgcnvn2crm9s24x2o43n45dyt8zbqq7artog1slvcea4hqme3jad2rc:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 42, "total_num_episodes": 55, "total_success_rate": 0.7636363636363637, "total_success_rate_lb_0.01": 0.6250906586396852, "total_success_rate_ub_0.01": 0.8734871148316137, "total_success_rate_lb_0.025": 0.6343281566828056, "total_success_rate_ub_0.025": 0.8572508766532698, "total_success_rate_lb_0.05": 0.6523377820544742, "total_success_rate_ub_0.05": 0.8484609349865173, "total_success_rate_lb_0.1": 0.6918580717954774, "total_success_rate_ub_0.1": 0.8336135632622587, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "542b00ca2276b525cfd3c1e781ad1cfa01a40d744351222703f194193dd1090c", "size": 520, "artifact_path": "wandb-client-artifact://uzg7fw6lpxv4783ez7tv6sr8w7af0ut8ocf8bm2qbeb9vv380plv0qyhv4vycvl7dpsiaf9yzb8dnxwo172tp82opgfk2yhe1zkwdj8hiiuavm03s57eiscyw0n35dhk/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://37fgfj5s7izz2ifbo7h5lq0epbdxrwmjv2xnt2btyvv46yu461wivw5vej9nwddrh3mkrcj0h4qlqulmer0u068wyvh9it68748gryqabje7ew0snrha14rsxrw2di9y:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 293}, "eval/total_success_rate": 0.7636363506317139, "eval/total_success_rate_lb_0.01": 0.6250906586647034, "eval/total_success_rate_ub_0.01": 0.873487114906311, "eval/total_success_rate_lb_0.025": 0.6343281269073486, "eval/total_success_rate_ub_0.025": 0.8572508692741394, "eval/total_success_rate_lb_0.05": 0.6523377895355225, "eval/total_success_rate_ub_0.05": 0.8484609127044678, "eval/total_success_rate_lb_0.1": 0.6918580532073975, "eval/total_success_rate_ub_0.1": 0.8336135745048523} \ No newline at end of file diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug-internal.log b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ee264f72d3b7b6386cd735accb031f07872a53f3 --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug-internal.log @@ -0,0 +1,809 @@ +2025-01-04 22:52:09,336 INFO StreamThr :2042123 [internal.py:wandb_internal():85] W&B internal server running at pid: 2042123, started at: 2025-01-04 22:52:09.336087 +2025-01-04 22:52:09,337 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:52:09,350 INFO WriterThread:2042123 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/run-xt2ne5el.wandb +2025-01-04 22:52:09,351 DEBUG SenderThread:2042123 [sender.py:send():379] send: header +2025-01-04 22:52:09,358 DEBUG SenderThread:2042123 [sender.py:send():379] send: run +2025-01-04 22:52:09,566 INFO SenderThread:2042123 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files +2025-01-04 22:52:09,566 INFO SenderThread:2042123 [sender.py:_start_run_threads():1188] run started: xt2ne5el with start time 1736049129.336554 +2025-01-04 22:52:09,581 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:52:09,581 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:52:09,628 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:52:09,633 DEBUG HandlerThread:2042123 [system_info.py:__init__():26] System info init +2025-01-04 22:52:09,633 DEBUG HandlerThread:2042123 [system_info.py:__init__():41] System info init done +2025-01-04 22:52:09,633 INFO HandlerThread:2042123 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:52:09,634 INFO SystemMonitor:2042123 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:52:09,634 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:52:09,634 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:52:09,635 INFO HandlerThread:2042123 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:52:09,635 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:52:09,636 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:52:09,636 INFO SystemMonitor:2042123 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:52:09,647 DEBUG HandlerThread:2042123 [system_info.py:probe():152] Probing system +2025-01-04 22:52:09,649 DEBUG HandlerThread:2042123 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_info.py:probe():200] Probing system done +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:52:09.647525', 'startedAt': '2025-01-05T03:52:09.324843', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.7805000000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4498.264, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.007, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 787.8546829223633}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:52:09,654 INFO HandlerThread:2042123 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:52:09,654 INFO HandlerThread:2042123 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:52:09,654 DEBUG HandlerThread:2042123 [system_info.py:_save_code():44] Saving code +2025-01-04 22:52:09,655 DEBUG HandlerThread:2042123 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:52:09,655 DEBUG HandlerThread:2042123 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:52:09,718 DEBUG HandlerThread:2042123 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:52:09,718 INFO HandlerThread:2042123 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:52:09,721 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:52:09,721 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:52:09,721 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:52:09,722 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:52:09,949 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/v8ig4z7x-wandb-metadata.json +2025-01-04 22:52:09,962 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:52:09,962 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:52:09,964 DEBUG SenderThread:2042123 [sender.py:send():379] send: telemetry +2025-01-04 22:52:10,002 INFO wandb-upload_2:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/dp777x6a-diff.patch +2025-01-04 22:52:10,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:10,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:10,005 INFO HandlerThread:2042123 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/tb" + save: true +} +control { + relay_id: "b91b4588e7fe4d55a0ffb1941cedd315" +} +_info { + stream_id: "xt2ne5el" +} + +2025-01-04 22:52:10,005 DEBUG HandlerThread:2042123 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:52:10,006 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:10,017 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/hwo6bjcq-code/FISH/eval_robot.py +2025-01-04 22:52:10,158 DEBUG SenderThread:2042123 [sender.py:send():379] send: tbrecord +2025-01-04 22:52:10,158 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:52:10,158 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 with policy live +2025-01-04 22:52:10,335 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/38td6c0n-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-metadata.json +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH/eval_robot.py +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/diff.patch +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:52:10,568 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code +2025-01-04 22:52:10,569 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH +2025-01-04 22:52:10,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:11,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:12,567 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:12,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:13,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:14,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:14,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:15,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:16,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:17,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:18,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:19,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:19,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:20,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:21,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:22,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:23,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:24,962 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:24,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:25,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:25,051 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:25,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:26,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:27,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:28,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:29,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:30,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:30,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:31,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:32,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:33,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:34,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:35,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:35,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:36,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:37,871 DEBUG SenderThread:2042123 [sender.py:send():379] send: config +2025-01-04 22:52:37,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:38,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:39,962 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:39,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:40,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:40,574 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:40,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:41,087 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:41,575 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml +2025-01-04 22:52:41,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:42,575 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:42,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:43,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:44,576 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:44,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:45,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:46,349 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:46,576 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:52:46,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:47,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:48,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:49,968 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:50,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:51,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:51,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:52,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:53,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:54,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:52:54,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:52:55,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:55,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:56,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:57,057 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:52:57,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:58,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:52:59,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:00,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:01,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:02,881 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:02,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:03,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:04,581 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:04,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:05,970 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:06,582 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:06,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:07,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:08,582 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:08,881 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:08,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:09,637 DEBUG SystemMonitor:2042123 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:53:09,638 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:53:09,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:09,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:09,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:10,972 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:11,972 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:12,583 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:12,973 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:13,973 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:14,461 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:14,584 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:14,973 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:15,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:16,584 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:16,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:17,981 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:18,981 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:19,466 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:19,981 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:20,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:21,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:22,586 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:22,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:23,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:24,963 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:24,963 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:25,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:25,071 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:25,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:26,984 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:27,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:28,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:29,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:30,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:30,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:31,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:32,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:33,782 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:53:33,782 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 with policy now +2025-01-04 22:53:33,783 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:53:33,784 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:53:33,784 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:53:33,787 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:53:33,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:34,066 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/cm9c9j2x-media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:53:34,589 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:34,589 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:53:34,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos +2025-01-04 22:53:34,864 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/7rspm7xz-media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:53:34,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:35,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:35,986 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:36,590 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:36,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:37,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:38,591 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:38,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:39,638 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:53:39,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:39,964 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:39,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:40,591 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:40,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:41,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:41,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:42,592 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:53:42,853 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wygl29c4-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:53:42,987 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:43,592 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:53:43,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:44,593 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:44,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:45,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:46,326 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:46,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:47,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:48,594 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:48,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:49,988 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:50,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:51,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:51,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:52,595 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:52,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:53,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:54,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:53:54,964 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:53:55,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:55,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:56,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:57,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:53:57,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:58,598 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:53:58,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:53:59,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:00,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:01,989 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:02,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:03,966 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:03,990 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:04,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:05,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:06,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:07,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:08,967 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:08,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:09,638 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:54:09,964 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:09,964 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:10,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:10,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:11,991 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:12,992 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:13,996 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:14,084 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:14,344 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:54:14,344 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 with policy now +2025-01-04 22:54:14,347 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:54:14,347 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:54:14,347 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:54:14,348 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:54:14,598 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/v9a0sdvp-media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:54:14,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:54:14,894 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/bifo173a-media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:54:14,999 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:15,996 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:16,603 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:17,001 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:18,001 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:18,604 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:18,998 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:19,813 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:20,008 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:20,604 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:20,997 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:22,008 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:22,605 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:23,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:23,606 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:54:24,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:24,440 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:54:24,441 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:54:24,441 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:54:24,606 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:54:24,606 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:24,974 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:24,976 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:24,976 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:25,017 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:26,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:26,607 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:27,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:28,003 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:28,607 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:29,003 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:30,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:30,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:31,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:32,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:33,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:34,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:35,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:35,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:36,004 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:37,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:38,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:38,611 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:39,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:39,639 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:54:39,965 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:39,965 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:40,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:41,005 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:41,068 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:42,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:43,006 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:44,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:45,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:46,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:46,281 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:46,613 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:47,007 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:48,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:49,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:50,009 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:51,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:51,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:52,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:53,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:54,010 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:54,971 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:54:54,972 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:54:55,012 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:55,257 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:54:55,257 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 with policy now +2025-01-04 22:54:55,258 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:54:55,258 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:54:55,259 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:54:55,261 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:54:55,519 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/ox298yj8-media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:54:55,616 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:54:55,616 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:54:55,617 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:54:55,886 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/e86uaaj7-media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:54:56,013 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:56,617 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:54:57,018 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:57,277 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:54:58,021 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:54:59,016 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:00,015 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:00,618 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:01,018 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:02,018 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:02,619 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:03,027 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:03,261 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:04,019 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:05,034 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:06,024 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:07,025 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:08,027 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:08,978 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:09,027 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:09,639 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:55:09,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:09,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:10,035 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:11,032 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:12,029 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:13,030 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:14,030 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:14,067 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:15,032 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:16,034 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:16,280 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:55:16,280 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:55:16,280 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:55:16,623 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:55:16,623 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:55:16,861 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wq2r75n7-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:55:17,041 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:18,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:18,624 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:19,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:19,345 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:20,043 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:21,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:22,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:22,625 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:23,036 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:24,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:24,631 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:24,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:24,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:25,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:26,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:27,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:28,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:29,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:29,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:30,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:31,037 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:32,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:33,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:34,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:34,977 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:35,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:36,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:37,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:38,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:39,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:39,640 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:55:39,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:39,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:40,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:40,065 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:41,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:42,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:43,038 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:44,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:45,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:45,978 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:46,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:47,039 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:48,045 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:48,428 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:55:48,428 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 with policy now +2025-01-04 22:55:48,430 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:55:48,431 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:55:48,431 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:55:48,431 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:55:48,641 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:55:48,641 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:48,641 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:55:48,642 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:55:48,687 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/lhcbg358-media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:55:48,945 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wf7yoy1l-media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:55:49,047 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:50,049 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:50,642 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:50,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:51,050 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:52,048 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:52,642 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:53,055 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:54,057 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:54,643 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:54,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:55:54,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:55:55,053 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:56,059 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:56,121 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:55:56,644 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:55:57,069 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:58,067 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:55:59,067 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:00,066 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:01,071 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:01,982 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:02,073 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:03,074 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:04,066 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:05,069 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:06,068 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:06,992 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:07,072 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:08,070 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:09,092 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:09,640 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:56:09,975 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:09,975 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:10,087 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:11,086 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:12,089 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:12,116 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:13,092 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:14,091 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:15,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:16,107 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:16,356 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:56:16,357 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:56:16,357 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:56:16,650 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:56:16,651 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:16,651 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:56:16,902 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/605inuye-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:56:17,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:17,861 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:18,107 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:18,651 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:19,100 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:20,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:20,652 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:21,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:22,096 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:22,653 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:22,924 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:23,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:24,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:24,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:24,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:25,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:26,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:27,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:28,055 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:28,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:29,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:30,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:31,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:32,097 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:33,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:33,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:34,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:35,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:36,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:37,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:38,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:38,985 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:39,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:39,641 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:56:39,969 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:39,969 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:40,098 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:41,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:42,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:43,099 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:44,100 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:44,128 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:45,101 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:46,100 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:47,101 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:48,101 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:49,103 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:49,440 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:49,703 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:56:49,703 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 with policy now +2025-01-04 22:56:49,704 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:56:49,704 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:56:49,704 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:56:49,705 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:56:49,963 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/q0wi346c-media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:56:50,102 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:50,662 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:56:50,662 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:50,663 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:56:50,663 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval +2025-01-04 22:56:50,956 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/do38pui6-media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:56:51,104 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:52,107 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:53,117 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:54,109 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:54,664 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:54,972 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:56:54,972 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:56:55,057 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:56:55,128 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:56,127 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:57,119 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:58,127 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:56:58,665 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:56:59,119 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:00,121 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:00,326 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:57:00,326 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:57:00,577 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:57:00,666 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:00,666 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:00,666 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:00,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media +2025-01-04 22:57:00,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval +2025-01-04 22:57:00,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table +2025-01-04 22:57:00,832 INFO wandb-upload_0:2042123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdjmvylrk +2025-01-04 22:57:00,843 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/j7ylrfmb-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:00,984 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:57:00,992 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:57:01,118 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:01,207 INFO SenderThread:2042123 [sender.py:send_request_log_artifact():1525] logged artifact run-xt2ne5el-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxMDc3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjAwNQ==', 'latestArtifact': None}} +2025-01-04 22:57:01,207 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:57:01,207 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 22:57:01,208 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:01,208 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,208 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:01,209 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,209 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:01,209 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,209 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,210 DEBUG SenderThread:2042123 [sender.py:send():379] send: telemetry +2025-01-04 22:57:01,210 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,210 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,210 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,210 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,211 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,212 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,213 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,213 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,213 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,214 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,214 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,214 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,214 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,215 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,215 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,215 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,215 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,216 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,216 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,216 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,216 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,217 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:01,217 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:57:01,446 INFO wandb-upload_2:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/wgf7axul-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:01,598 INFO wandb-upload_1:2042123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp49l7jzj8 +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:01,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval +2025-01-04 22:57:01,970 INFO SenderThread:2042123 [sender.py:send_request_log_artifact():1525] logged artifact run-xt2ne5el-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTIxMDc4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2NjAwNg==', 'latestArtifact': None}} +2025-01-04 22:57:01,970 DEBUG SenderThread:2042123 [sender.py:send():379] send: files +2025-01-04 22:57:01,971 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json with policy now +2025-01-04 22:57:01,971 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:01,971 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:01,971 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:02,118 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:02,142 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/uown77oo-media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:02,386 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/itemdwsk-media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:02,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:02,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:03,118 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:03,273 DEBUG SenderThread:2042123 [sender.py:send():379] send: exit +2025-01-04 22:57:03,273 INFO SenderThread:2042123 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:send_exit():588] handling runtime: 293 +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:send_exit():594] send defer +2025-01-04 22:57:03,274 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,274 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:57:03,274 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:57:03,274 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:57:03,275 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:57:03,275 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,275 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:57:03,275 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:57:03,275 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:57:03,275 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:57:03,276 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:57:03,276 DEBUG SystemMonitor:2042123 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:57:03,276 DEBUG SystemMonitor:2042123 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:57:03,327 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:57:03,328 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:57:03,328 INFO HandlerThread:2042123 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:57:03,328 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,328 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:57:03,328 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:57:03,328 DEBUG SenderThread:2042123 [sender.py:send():379] send: stats +2025-01-04 22:57:03,329 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,329 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:57:03,329 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:03,329 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:57:03,329 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:57:03,329 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:03,329 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:57:03,667 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:04,668 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:09,420 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:09,420 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:09,420 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:57:09,421 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,421 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:57:09,421 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:57:09,421 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:09,422 DEBUG SenderThread:2042123 [sender.py:send():379] send: history +2025-01-04 22:57:09,422 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:57:09,422 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:09,422 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:57:09,423 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:09,423 DEBUG SenderThread:2042123 [sender.py:send():379] send: summary +2025-01-04 22:57:09,426 INFO SenderThread:2042123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:57:09,426 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,427 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:57:09,427 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:57:09,427 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:09,427 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:57:09,427 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,427 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:57:09,431 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:57:09,529 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:57:09,529 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:09,529 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:57:09,530 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:09,530 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:57:09,670 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:09,670 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml +2025-01-04 22:57:10,274 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:10,472 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:57:10,472 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:10,472 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:10,472 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:57:10,473 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:10,473 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:57:10,473 INFO SenderThread:2042123 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:57:10,475 INFO SenderThread:2042123 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:57:10,477 INFO SenderThread:2042123 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:57:10,489 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:57:10,490 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:10,490 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:57:10,490 DEBUG SenderThread:2042123 [sender.py:send():379] send: artifact +2025-01-04 22:57:10,655 INFO SenderThread:2042123 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:57:10,655 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:10,656 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:57:10,656 INFO SenderThread:2042123 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:57:10,670 INFO Thread-12 :2042123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml config.yaml +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json wandb-summary.json +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt requirements.txt +2025-01-04 22:57:10,671 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:57:10,672 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log output.log +2025-01-04 22:57:10,672 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/diff.patch diff.patch +2025-01-04 22:57:10,675 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 media/videos/eval/1_eval_1_537b6b4a557eb4c5962a.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 media/videos/eval/0_eval_0_fa08c14e432d2c4b05ac.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 media/videos/eval/4_eval_4_bf759c4daff03b0779d3.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 media/videos/eval/2_eval_2_6556ec2efb8093719953.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 media/videos/eval/3_eval_3_55177e1b719e0936946e.mp4 +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json media/table/eval/total_success_rate_ci_6_542b00ca2276b525cfd3.table.json +2025-01-04 22:57:10,680 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:57:10,681 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:10,681 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:57:10,681 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:10,681 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:57:10,681 INFO SenderThread:2042123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:57:10,858 INFO wandb-upload_0:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/config.yaml +2025-01-04 22:57:10,893 INFO wandb-upload_2:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/wandb-summary.json +2025-01-04 22:57:10,896 INFO wandb-upload_3:2042123 [upload_job.py:push():130] Uploaded file /tmp/tmps46638pxwandb/yx64pkx2-events.out.tfevents.1736049129.leonmkim-ROG-Strix-G15CS-G15CS.2041941.0 +2025-01-04 22:57:10,902 INFO wandb-upload_1:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/requirements.txt +2025-01-04 22:57:10,913 INFO wandb-upload_4:2042123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/files/output.log +2025-01-04 22:57:11,114 INFO Thread-11 (_thread_body):2042123 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:57:11,114 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,114 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:57:11,114 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,114 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:57:11,114 INFO SenderThread:2042123 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:57:11,117 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:57:11,117 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,117 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:57:11,117 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,117 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:57:11,117 INFO SenderThread:2042123 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:57:11,275 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:11,342 INFO SenderThread:2042123 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:57:11,342 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:57:11,342 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:11,342 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,342 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,343 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:57:11,343 INFO SenderThread:2042123 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:57:11,343 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:57:11,343 INFO HandlerThread:2042123 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send():379] send: final +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send():379] send: footer +2025-01-04 22:57:11,343 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: defer +2025-01-04 22:57:11,343 INFO SenderThread:2042123 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:57:11,344 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:11,344 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:57:11,344 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:11,344 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:57:11,346 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:57:11,346 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:57:11,346 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:57:11,347 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:57:11,398 DEBUG SenderThread:2042123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:57:11,398 INFO MainThread:2042123 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:57:11,399 INFO MainThread:2042123 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:57:11,399 INFO MainThread:2042123 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:57:11,399 DEBUG HandlerThread:2042123 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:57:11,399 INFO HandlerThread:2042123 [handler.py:finish():882] shutting down handler +2025-01-04 22:57:12,346 INFO WriterThread:2042123 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/run-xt2ne5el.wandb +2025-01-04 22:57:12,398 INFO SenderThread:2042123 [sender.py:finish():1615] shutting down sender +2025-01-04 22:57:12,398 INFO SenderThread:2042123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:57:12,398 INFO SenderThread:2042123 [file_pusher.py:join():175] waiting for file pusher diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug.log b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ac96bc5b961cedceaf58c28284b68e6c37bd897c --- /dev/null +++ b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Configure stats pid to 2041941 +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/settings +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug.log +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/logs/debug-internal.log +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el'} +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():619] starting backend +2025-01-04 22:52:09,331 INFO MainThread:2041941 [wandb_init.py:init():623] setting up manager +2025-01-04 22:52:09,335 INFO MainThread:2041941 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:52:09,336 INFO MainThread:2041941 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:52:09,348 INFO MainThread:2041941 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:52:09,356 INFO MainThread:2041941 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:52:09,581 INFO MainThread:2041941 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:52:09,623 INFO MainThread:2041941 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:52:09,623 INFO MainThread:2041941 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:52:09,962 INFO MainThread:2041941 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:52:09,963 INFO MainThread:2041941 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:52:09,963 INFO MainThread:2041941 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:52:09,963 INFO MainThread:2041941 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:52:09,965 INFO MainThread:2041941 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:52:09,965 INFO MainThread:2041941 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/xt2ne5el/tb, True +2025-01-04 22:52:37,870 INFO MainThread:2041941 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:57:12,400 WARNING MsgRouterThr:2041941 [router.py:message_loop():77] message_loop has been closed diff --git a/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/run-xt2ne5el.wandb b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/run-xt2ne5el.wandb new file mode 100644 index 0000000000000000000000000000000000000000..53726372deca681f76d6507d1bc83026cdc6d7aa Binary files /dev/null and b/xt2ne5el/wandb/run-20250104_225209-xt2ne5el/run-xt2ne5el.wandb differ