diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c2762adbf07f553b242ddbed308eec7aabe4b651 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,48 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/episode_rosbags/episode_1_2025-01-04-21-10-39.bag filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ukc5qfsm/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ot5wit9y/episode_rosbags/episode_2_2025-01-04-21-11-30.bag filter=lfs diff=lfs merge=lfs -text +ot5wit9y/episode_rosbags/episode_3_2025-01-04-21-12-11.bag filter=lfs diff=lfs merge=lfs -text +ot5wit9y/episode_rosbags/episode_0_2025-01-04-21-09-58.bag filter=lfs diff=lfs merge=lfs -text +ot5wit9y/episode_rosbags/episode_4_2025-01-04-21-12-56.bag filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +6mzw1wsh/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/6mzw1wsh/.hydra/hydra.yaml b/6mzw1wsh/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..968ad802eb25cb6e5c098ee508c52a069e30c358 --- /dev/null +++ b/6mzw1wsh/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/6mzw1wsh/.hydra/overrides.yaml b/6mzw1wsh/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/6mzw1wsh/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/6mzw1wsh/episode_rosbags/aligned_depth_to_color_K.npy b/6mzw1wsh/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/6mzw1wsh/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/6mzw1wsh/episode_rosbags/cam_tf_world.npy b/6mzw1wsh/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/6mzw1wsh/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/6mzw1wsh/episode_rosbags/color_K.npy b/6mzw1wsh/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/6mzw1wsh/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/6mzw1wsh/episode_rosbags/depth_K.npy b/6mzw1wsh/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/6mzw1wsh/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/6mzw1wsh/eval_robot.log b/6mzw1wsh/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..1250adbea5e3aca0ccb9538bfb55a88d2ec85355 --- /dev/null +++ b/6mzw1wsh/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:23:03,659][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:23:03,662][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/6mzw1wsh/eval_video/0_eval.mp4 b/6mzw1wsh/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0dc5ac1d0b95ca5f8022b28e611b2df76dcdf113 --- /dev/null +++ b/6mzw1wsh/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:204e5f6d515db78c9797f141a381290f375d6efe7941034acf7efa403cedd34b +size 1138629 diff --git a/6mzw1wsh/eval_video/1_eval.mp4 b/6mzw1wsh/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df0a75813de2a1700725db6f5ab10cf0f9d5334a --- /dev/null +++ b/6mzw1wsh/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39b06e75cb8443c40e412577a5428c4a526c986e6499a341b7bc0f1565e98fc +size 1133959 diff --git a/6mzw1wsh/eval_video/2_eval.mp4 b/6mzw1wsh/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7d85aa77cd97a31f8fe173ca790e0fc380c1e50 --- /dev/null +++ b/6mzw1wsh/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b75c49234e97b158f8f0c9df591cd99b39e05b03a836b1e628f578caf03c4c +size 1138515 diff --git a/6mzw1wsh/eval_video/3_eval.mp4 b/6mzw1wsh/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c8b57beefb7def18dad5bfc4b579a926011f853 --- /dev/null +++ b/6mzw1wsh/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75933aeb4ef7e707523abcd74cbddf7203aa7422728f68ebceee9b7883b70d2d +size 1175309 diff --git a/6mzw1wsh/eval_video/4_eval.mp4 b/6mzw1wsh/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7cad159a97478aa9ad5b5e9aba5debf4a7704631 --- /dev/null +++ b/6mzw1wsh/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7d77604e2e08eabb6333a7a8a0ea6f062a198b092e5f6e6eb38cba554f1d3ce +size 1138986 diff --git a/6mzw1wsh/tb/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 b/6mzw1wsh/tb/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 new file mode 100644 index 0000000000000000000000000000000000000000..1d685afc803be6db33b1b6e694089d8949106777 --- /dev/null +++ b/6mzw1wsh/tb/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d5915878afcddba90443b2e54d4d70b1912b820198817a5421a656884eab22 +size 6539 diff --git a/6mzw1wsh/wandb/debug-internal.log b/6mzw1wsh/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e1c754418a7ffa1c6038f6143c9d2d5f8672bea8 --- /dev/null +++ b/6mzw1wsh/wandb/debug-internal.log @@ -0,0 +1,740 @@ +2025-01-03 22:23:09,363 INFO StreamThr :1982600 [internal.py:wandb_internal():85] W&B internal server running at pid: 1982600, started at: 2025-01-03 22:23:09.363010 +2025-01-03 22:23:09,364 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:23:09,375 INFO WriterThread:1982600 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/run-6mzw1wsh.wandb +2025-01-03 22:23:09,375 DEBUG SenderThread:1982600 [sender.py:send():379] send: header +2025-01-03 22:23:09,382 DEBUG SenderThread:1982600 [sender.py:send():379] send: run +2025-01-03 22:23:09,590 INFO SenderThread:1982600 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files +2025-01-03 22:23:09,590 INFO SenderThread:1982600 [sender.py:_start_run_threads():1188] run started: 6mzw1wsh with start time 1735960989.363361 +2025-01-03 22:23:09,606 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:23:09,606 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:23:09,676 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:23:09,681 DEBUG HandlerThread:1982600 [system_info.py:__init__():26] System info init +2025-01-03 22:23:09,681 DEBUG HandlerThread:1982600 [system_info.py:__init__():41] System info init done +2025-01-03 22:23:09,681 INFO HandlerThread:1982600 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:23:09,681 INFO SystemMonitor:1982600 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:23:09,681 INFO HandlerThread:1982600 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:23:09,682 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:23:09,682 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:23:09,683 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:23:09,683 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:23:09,684 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:23:09,694 DEBUG HandlerThread:1982600 [system_info.py:probe():152] Probing system +2025-01-03 22:23:09,695 DEBUG HandlerThread:1982600 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_info.py:probe():200] Probing system done +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:23:09.694174', 'startedAt': '2025-01-04T03:23:09.354185', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4320.156875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4517.955, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4510.961, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 838.2302551269531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:23:09,700 INFO HandlerThread:1982600 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:23:09,700 INFO HandlerThread:1982600 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_info.py:_save_code():44] Saving code +2025-01-03 22:23:09,701 DEBUG HandlerThread:1982600 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:23:09,701 DEBUG HandlerThread:1982600 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:23:09,764 DEBUG HandlerThread:1982600 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:23:09,764 INFO HandlerThread:1982600 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:23:09,767 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:23:09,767 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:23:09,767 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:23:09,768 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:23:10,000 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:23:10,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:23:10,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:10,002 DEBUG SenderThread:1982600 [sender.py:send():379] send: telemetry +2025-01-03 22:23:10,019 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/pw0qsnwy-wandb-metadata.json +2025-01-03 22:23:10,048 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:10,048 INFO HandlerThread:1982600 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/tb" + save: true +} +control { + relay_id: "e231d7849ef446b99386d81c71cff138" +} +_info { + stream_id: "6mzw1wsh" +} + +2025-01-03 22:23:10,049 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:10,050 DEBUG HandlerThread:1982600 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:23:10,067 INFO wandb-upload_2:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/1cvmaba9-diff.patch +2025-01-03 22:23:10,079 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/86ump4oz-code/FISH/eval_robot.py +2025-01-03 22:23:10,177 DEBUG SenderThread:1982600 [sender.py:send():379] send: tbrecord +2025-01-03 22:23:10,177 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:23:10,177 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 with policy live +2025-01-03 22:23:10,375 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/w56zewy7-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH/eval_robot.py +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-metadata.json +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/diff.patch +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:10,593 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code +2025-01-03 22:23:10,593 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH +2025-01-03 22:23:11,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:12,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:12,591 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:13,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:14,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:15,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:15,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:16,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:17,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:18,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:19,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:20,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:20,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:21,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:22,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:22,375 DEBUG SenderThread:1982600 [sender.py:send():379] send: config +2025-01-03 22:23:23,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:24,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:24,594 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:25,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:25,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:25,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:25,093 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:26,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:27,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:28,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:28,595 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:29,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:30,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:31,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:31,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:32,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:32,596 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:33,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:34,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:35,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:36,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:36,473 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:36,597 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:37,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:38,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:38,598 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:39,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:40,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:40,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:40,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:40,598 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:41,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:42,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:42,086 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:42,599 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml +2025-01-03 22:23:43,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:44,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:45,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:46,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:47,008 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:48,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:48,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:49,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:50,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:51,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:52,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:53,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:54,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:54,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:55,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:55,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:55,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:56,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:57,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:58,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:59,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:59,121 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:00,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:01,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:02,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:03,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:04,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:05,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:05,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:06,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:06,605 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:07,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:07,185 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:24:07,186 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:24:07,186 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 with policy now +2025-01-03 22:24:07,186 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:24:07,186 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:24:07,187 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:24:07,456 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/rb74u4is-media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:24:07,605 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:24:08,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:08,027 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/6czpcucr-media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:24:08,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:09,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:09,684 DEBUG SystemMonitor:1982600 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:24:09,685 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:24:10,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:10,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:10,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:10,072 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:11,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:12,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:12,607 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:13,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:14,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:14,608 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:15,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:16,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:16,013 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:16,609 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:24:16,861 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/fyy195pm-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:24:17,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:18,013 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:18,610 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:19,013 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:20,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:21,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:21,250 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:22,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:22,611 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:23,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:24,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:25,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:25,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:25,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:26,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:27,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:27,084 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:28,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:29,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:30,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:31,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:32,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:33,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:33,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:34,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:35,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:36,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:37,016 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:38,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:38,016 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:39,018 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:39,685 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:24:40,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:40,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:40,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:41,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:42,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:43,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:43,088 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:44,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:45,021 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:46,021 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:47,022 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:48,022 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:48,279 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:48,361 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:24:48,361 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 with policy now +2025-01-03 22:24:48,363 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:24:48,364 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:24:48,364 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:24:48,364 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:24:48,619 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:48,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:24:48,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:24:48,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:24:48,627 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/pnzyxy9r-media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:24:48,881 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/bd9fft1d-media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:24:49,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:50,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:50,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:51,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:52,029 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:53,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:54,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:54,121 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:54,621 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:55,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:55,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:55,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:56,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:56,622 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:57,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:58,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:58,479 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:24:58,480 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:24:58,481 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:24:58,623 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:24:58,623 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:24:59,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:00,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:00,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:00,624 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:01,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:02,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:02,624 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:03,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:04,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:04,625 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:05,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:06,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:06,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:07,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:08,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:09,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:09,686 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:25:10,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:10,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:10,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:11,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:11,164 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:12,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:13,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:14,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:15,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:16,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:17,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:17,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:18,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:19,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:20,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:21,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:22,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:22,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:23,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:24,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:25,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:25,004 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:25,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:26,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:27,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:27,081 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:28,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:29,032 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:30,037 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:30,136 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:25:30,136 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 with policy now +2025-01-03 22:25:30,138 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:25:30,139 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:25:30,139 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:25:30,140 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:25:30,418 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/njth5p9o-media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:25:30,634 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:30,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:25:30,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:25:30,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:25:30,904 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/38q3h014-media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:25:31,035 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:32,036 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:32,162 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:32,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:33,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:34,043 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:35,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:36,044 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:36,637 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:37,037 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:37,649 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:38,039 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:38,637 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:39,038 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:39,686 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:25:40,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:40,005 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:40,039 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:40,308 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:25:40,309 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:25:40,310 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:25:40,638 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:40,638 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:25:40,638 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:25:40,897 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/0rjd4ust-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:25:41,049 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:42,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:42,639 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:42,885 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:43,044 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:44,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:44,639 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:45,053 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:46,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:46,640 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:47,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:48,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:48,054 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:49,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:50,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:51,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:52,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:53,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:54,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:54,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:55,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:55,004 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:55,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:56,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:57,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:58,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:59,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:59,072 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:00,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:01,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:02,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:03,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:04,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:04,401 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:04,646 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:05,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:06,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:07,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:08,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:08,647 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:09,043 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:09,687 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:26:09,687 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:10,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:10,004 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:10,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:11,043 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:12,046 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:12,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:13,046 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:13,053 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:26:13,053 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 with policy now +2025-01-03 22:26:13,097 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:26:13,098 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:26:13,098 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:26:13,098 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:26:13,327 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/7zcazti4-media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:26:13,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:26:13,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:26:13,650 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:26:13,917 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/ef69g7vh-media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:26:14,049 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:14,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:15,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:15,048 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:16,051 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:17,056 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:18,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:18,651 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:19,061 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:20,048 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:20,651 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:21,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:21,051 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:22,051 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:23,053 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:23,652 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:26:24,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:24,363 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:26:24,363 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:26:24,364 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:26:24,653 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:24,653 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:26:25,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:25,007 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:25,060 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:26,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:26,117 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:26,654 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:27,059 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:28,054 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:28,654 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:29,054 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:30,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:31,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:32,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:32,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:33,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:34,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:35,056 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:36,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:37,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:37,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:38,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:39,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:39,687 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:26:40,006 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:40,006 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:40,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:41,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:42,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:42,122 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:43,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:44,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:45,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:46,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:47,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:48,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:48,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:49,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:50,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:51,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:52,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:53,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:53,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:54,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:55,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:55,012 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:55,067 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:55,405 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:26:55,405 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 with policy now +2025-01-03 22:26:55,407 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:26:55,407 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:26:55,407 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:26:55,410 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:26:55,656 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/958ilpfd-media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:26:55,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:26:55,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:26:55,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:26:55,932 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/qysku1ms-media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:26:56,061 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:56,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:57,065 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:58,068 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:58,421 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:59,069 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:00,073 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:00,665 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:01,072 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:02,070 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:03,075 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:04,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:04,075 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:04,667 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:05,071 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:06,073 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:07,081 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:08,079 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:09,078 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:09,687 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:27:09,688 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:10,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:27:10,010 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:27:10,079 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:11,080 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:12,080 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:13,080 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:14,090 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:15,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:15,093 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:15,389 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:27:15,390 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:27:15,539 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval +2025-01-03 22:27:15,860 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/oacesd5o-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:15,949 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/o2zlq7tq-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:16,041 INFO wandb-upload_2:1982600 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpiya1qizf +2025-01-03 22:27:16,077 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:16,454 INFO SenderThread:1982600 [sender.py:send_request_log_artifact():1525] logged artifact run-6mzw1wsh-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3MzU2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4Nzg0OQ==', 'latestArtifact': None}} +2025-01-03 22:27:16,454 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:16,454 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,455 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,455 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:27:16,455 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-03 22:27:16,455 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:16,455 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,459 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,459 DEBUG SenderThread:1982600 [sender.py:send():379] send: telemetry +2025-01-03 22:27:16,459 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,460 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,460 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,460 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,460 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,460 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,461 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,461 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,461 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,461 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,462 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,462 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,462 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,462 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,462 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,463 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,463 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,463 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,463 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,464 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,464 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,464 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:16,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:16,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:16,713 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/lho5mtpi-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:17,078 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:17,370 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:27:17,370 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:27:17,413 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:27:17,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:17,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval +2025-01-03 22:27:17,752 INFO wandb-upload_2:1982600 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptu9wmfj3 +2025-01-03 22:27:17,856 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/9jvdau5e-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:18,078 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:18,261 INFO SenderThread:1982600 [sender.py:send_request_log_artifact():1525] logged artifact run-6mzw1wsh-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3MzYwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4Nzg1Ng==', 'latestArtifact': None}} +2025-01-03 22:27:18,261 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:27:18,261 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-03 22:27:18,262 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:18,262 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:18,262 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:18,498 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/gdm6fide-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:18,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:18,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:18,877 DEBUG SenderThread:1982600 [sender.py:send():379] send: exit +2025-01-03 22:27:18,877 INFO SenderThread:1982600 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:27:18,877 INFO SenderThread:1982600 [sender.py:send_exit():588] handling runtime: 249 +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:send_exit():594] send defer +2025-01-03 22:27:18,878 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,878 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:27:18,878 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:27:18,878 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,879 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:27:18,879 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,879 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:27:18,879 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:27:18,879 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,879 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:27:18,879 INFO HandlerThread:1982600 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:27:18,879 DEBUG SystemMonitor:1982600 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:27:18,879 DEBUG SystemMonitor:1982600 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:27:18,880 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:27:18,880 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:27:18,934 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:27:18,934 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:27:18,934 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:27:18,934 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,934 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:27:18,934 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:27:18,934 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:27:18,935 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,935 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:27:18,935 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,935 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:27:18,935 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:27:18,935 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,935 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:27:19,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:20,673 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:25,435 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:25,435 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:25,435 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:27:25,436 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,436 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:27:25,436 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:27:25,436 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:25,436 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:25,437 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:27:25,437 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:25,437 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:25,438 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:25,438 DEBUG SenderThread:1982600 [sender.py:send():379] send: summary +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:25,442 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:27:25,442 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:25,442 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:27:25,442 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:27:25,446 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:25,572 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:27:25,572 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:25,572 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:27:25,572 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,572 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:27:25,675 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml +2025-01-03 22:27:25,675 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:25,878 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:28,439 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:27:28,439 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:28,440 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:28,440 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:27:28,440 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:28,440 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:27:28,440 INFO SenderThread:1982600 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:27:28,442 INFO SenderThread:1982600 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:27:28,444 INFO SenderThread:1982600 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:27:28,456 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:27:28,456 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:28,457 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:27:28,457 DEBUG SenderThread:1982600 [sender.py:send():379] send: artifact +2025-01-03 22:27:28,663 INFO SenderThread:1982600 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:27:28,663 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:28,664 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:27:28,664 INFO SenderThread:1982600 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:27:28,676 INFO SenderThread:1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:28,676 INFO SenderThread:1982600 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml config.yaml +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json wandb-summary.json +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt requirements.txt +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log output.log +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/diff.patch diff.patch +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:27:28,684 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:28,684 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:28,685 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:27:28,685 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:28,686 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:27:28,686 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:28,686 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:27:28,686 INFO SenderThread:1982600 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:27:28,865 INFO wandb-upload_2:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:28,876 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml +2025-01-03 22:27:28,880 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:28,880 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:28,896 INFO wandb-upload_4:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:28,901 INFO wandb-upload_3:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt +2025-01-03 22:27:28,993 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/fb5fzend-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:29,194 INFO Thread-11 (_thread_body):1982600 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:27:29,194 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,194 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:27:29,194 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,194 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:27:29,194 INFO SenderThread:1982600 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:27:29,197 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:27:29,198 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,198 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:27:29,198 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,198 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:27:29,198 INFO SenderThread:1982600 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:27:29,391 INFO SenderThread:1982600 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:27:29,392 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:27:29,392 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,392 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:27:29,392 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,392 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:27:29,392 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:27:29,392 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,393 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:27:29,393 DEBUG SenderThread:1982600 [sender.py:send():379] send: final +2025-01-03 22:27:29,393 DEBUG SenderThread:1982600 [sender.py:send():379] send: footer +2025-01-03 22:27:29,393 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,393 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:27:29,393 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:29,394 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:27:29,394 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:29,394 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:27:29,395 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:27:29,395 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:29,395 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:29,395 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:27:29,450 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:29,450 INFO MainThread:1982600 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:27:29,450 INFO MainThread:1982600 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:27:29,451 INFO MainThread:1982600 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:27:29,451 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:27:29,451 INFO HandlerThread:1982600 [handler.py:finish():882] shutting down handler +2025-01-03 22:27:30,394 INFO WriterThread:1982600 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/run-6mzw1wsh.wandb +2025-01-03 22:27:30,450 INFO SenderThread:1982600 [sender.py:finish():1615] shutting down sender +2025-01-03 22:27:30,450 INFO SenderThread:1982600 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:27:30,450 INFO SenderThread:1982600 [file_pusher.py:join():175] waiting for file pusher diff --git a/6mzw1wsh/wandb/debug.log b/6mzw1wsh/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2221bf96552f9d18ff08fb796a8da1404bd0be9b --- /dev/null +++ b/6mzw1wsh/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Configure stats pid to 1982433 +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/settings +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug.log +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug-internal.log +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh'} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():619] starting backend +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():623] setting up manager +2025-01-03 22:23:09,362 INFO MainThread:1982433 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:23:09,363 INFO MainThread:1982433 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:23:09,374 INFO MainThread:1982433 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:23:09,380 INFO MainThread:1982433 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:23:09,605 INFO MainThread:1982433 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:23:09,671 INFO MainThread:1982433 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:23:09,671 INFO MainThread:1982433 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:23:10,003 INFO MainThread:1982433 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:23:10,004 INFO MainThread:1982433 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/tb, True +2025-01-03 22:23:22,375 INFO MainThread:1982433 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:27:30,452 WARNING MsgRouterThr:1982433 [router.py:message_loop():77] message_loop has been closed diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH/eval_robot.py b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d8b88ea9e9f4f76f6fbe2749ac8011b77aa930c --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735960989 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/diff.patch b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json new file mode 100644 index 0000000000000000000000000000000000000000..db88f8dce313fe228bb6c4839d50146f71aa6144 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 20, 1.0, 0.9946215747797684, 1, 0.8453640490079612, 1, 0.865594230519132, 1, 1, 1]]} \ No newline at end of file diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0dc5ac1d0b95ca5f8022b28e611b2df76dcdf113 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:204e5f6d515db78c9797f141a381290f375d6efe7941034acf7efa403cedd34b +size 1138629 diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df0a75813de2a1700725db6f5ab10cf0f9d5334a --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39b06e75cb8443c40e412577a5428c4a526c986e6499a341b7bc0f1565e98fc +size 1133959 diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7d85aa77cd97a31f8fe173ca790e0fc380c1e50 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b75c49234e97b158f8f0c9df591cd99b39e05b03a836b1e628f578caf03c4c +size 1138515 diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c8b57beefb7def18dad5bfc4b579a926011f853 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75933aeb4ef7e707523abcd74cbddf7203aa7422728f68ebceee9b7883b70d2d +size 1175309 diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7cad159a97478aa9ad5b5e9aba5debf4a7704631 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7d77604e2e08eabb6333a7a8a0ea6f062a198b092e5f6e6eb38cba554f1d3ce +size 1138986 diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dd7470ff227f963d7ed837e098185b6d95c3be59 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log @@ -0,0 +1,271 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735961002.375766]: resetting environment +[INFO] [1735961002.378970]: cleared current plan +[INFO] [1735961002.379888]: moving to home +[INFO] [1735961003.382585]: reached home +[INFO] [1735961003.384251]: reset action history +[INFO] [1735961006.093617]: environment reset +Starting episode 0 +[INFO] [1735961006.095716]: resetting environment +[INFO] [1735961006.098347]: cleared current plan +[INFO] [1735961006.099438]: moving to home +[INFO] [1735961007.101534]: reached home +[INFO] [1735961007.103103]: reset action history +[INFO] [1735961009.813080]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735961015.463237]: resetting environment +[INFO] [1735961015.466270]: cleared current plan +[INFO] [1735961015.470122]: moving to home +[INFO] [1735961016.472852]: reached home +[INFO] [1735961016.474410]: reset action history +[INFO] [1735961019.181633]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/episode_rosbags/episode_0_2025-01-03-22-23-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1982433.log +started roslaunch server http://158.130.50.37:38191/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1982700] +process[print_text-2]: started with pid [1982724] +process[rosbag_record-3]: started with pid [1982725] +started bagging! +For topic gripper_width: timestamp difference is -263889300 for nearest: 1735961043661532415 - target: 1735961043925421715 at idx 249 +For topic gripper_width: timestamp difference is 104757374 for nearest: 1735961045028151339 - target: 1735961044923393965 at idx 249 +For topic gripper_width: timestamp difference is 101418433 for nearest: 1735961046028207478 - target: 1735961045926789045 at idx 249 +[INFO] [1735961046.201391]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735961046.927437]: Stored episode 1. +[INFO] [1735961046.928225]: Saving video... +[INFO] [1735961047.139928]: Video saved! +Starting episode 1 +[INFO] [1735961047.160201]: resetting environment +[INFO] [1735961047.166361]: cleared current plan +[INFO] [1735961047.166569]: moving to home +[INFO] [1735961050.869028]: reached home +[INFO] [1735961050.869483]: reset action history +[INFO] [1735961053.578302]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961056.284677]: resetting environment +[INFO] [1735961056.287888]: cleared current plan +[INFO] [1735961056.288281]: moving to home +[INFO] [1735961057.289271]: reached home +[INFO] [1735961057.289833]: reset action history +[INFO] [1735961059.997188]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/episode_rosbags/episode_1_2025-01-03-22-24-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1982433.log +started roslaunch server http://158.130.50.37:45175/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1982835] +process[print_text-5]: started with pid [1982836] +process[rosbag_record-6]: started with pid [1982860] +started bagging! +[INFO] [1735961086.962961]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit[pub_text-4] killing on exit +[INFO] [1735961087.724785]: Stored episode 2. +[INFO] [1735961087.724998]: Saving video... +[WARN] [1735961088.278582]: Plan exhausted +[WARN] [1735961088.312463]: Plan exhausted +[WARN] [1735961088.352920]: Plan exhausted +[INFO] [1735961088.325318]: Video saved! +Starting episode 2 +[INFO] [1735961088.362970]: resetting environment +[INFO] [1735961088.371983]: cleared current plan +[INFO] [1735961088.372513]: moving to home +[INFO] [1735961092.077174]: reached home +[INFO] [1735961092.078620]: reset action history +[INFO] [1735961094.846364]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961098.054067]: resetting environment +[INFO] [1735961098.062610]: cleared current plan +[INFO] [1735961098.063148]: moving to home +[INFO] [1735961099.065316]: reached home +[INFO] [1735961099.065841]: reset action history +[INFO] [1735961101.777548]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/episode_rosbags/episode_2_2025-01-03-22-25-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1982433.log +started roslaunch server http://158.130.50.37:42285/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1982971] +process[print_text-8]: started with pid [1982972] +process[rosbag_record-9]: started with pid [1982996] +started bagging! +[WARN] [1735961130.070993]: Plan exhausted +[WARN] [1735961130.110970]: Plan exhausted +[WARN] [1735961130.150521]: Plan exhausted +[INFO] [1735961128.748246]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1735961129.520186]: Stored episode 3. +[INFO] [1735961129.523882]: Saving video... +[INFO] [1735961130.099531]: Video saved! +Starting episode 3 +[INFO] [1735961130.140117]: resetting environment +[INFO] [1735961130.156458]: cleared current plan +[INFO] [1735961130.160843]: moving to home +[INFO] [1735961133.874251]: reached home +[INFO] [1735961133.877194]: reset action history +[INFO] [1735961136.605615]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961139.819259]: resetting environment +[INFO] [1735961139.837888]: cleared current plan +[INFO] [1735961139.839113]: moving to home +[INFO] [1735961140.841572]: reached home +[INFO] [1735961140.843490]: reset action history +[INFO] [1735961143.794642]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/episode_rosbags/episode_3_2025-01-03-22-25-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1982433.log +started roslaunch server http://158.130.50.37:43689/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1983130] +process[print_text-11]: started with pid [1983131] +process[rosbag_record-12]: started with pid [1983132] +started bagging! +For topic gripper_width: timestamp difference is -105490778 for nearest: 1735961163161534216 - target: 1735961163267024994 at idx 249 +For topic gripper_width: timestamp difference is -104175127 for nearest: 1735961165961535418 - target: 1735961166065710545 at idx 249 +[INFO] [1735961171.730544]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[WARN] [1735961173.038198]: Plan exhausted +[INFO] [1735961172.497852]: Stored episode 4. +[INFO] [1735961172.505295]: Saving video... +[INFO] [1735961172.997761]: Video saved! +Starting episode 4 +[INFO] [1735961173.057246]: resetting environment +[INFO] [1735961173.068296]: cleared current plan +[INFO] [1735961173.068613]: moving to home +[INFO] [1735961176.772860]: reached home +[INFO] [1735961176.776395]: reset action history +[INFO] [1735961179.576125]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961183.575135]: resetting environment +[INFO] [1735961183.591179]: cleared current plan +[INFO] [1735961183.594535]: moving to home +[INFO] [1735961184.598328]: reached home +[INFO] [1735961184.603004]: reset action history +[INFO] [1735961187.379576]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/episode_rosbags/episode_4_2025-01-03-22-26-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1982433.log +started roslaunch server http://158.130.50.37:36695/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1983242] +process[print_text-14]: started with pid [1983243] +process[rosbag_record-15]: started with pid [1983267] +started bagging! +[INFO] [1735961214.365970]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735961215.140238]: Stored episode 5. +[INFO] [1735961215.142791]: Saving video... +[INFO] [1735961215.363961]: Video saved! +[INFO] [1735961215.407609]: resetting environment +[INFO] [1735961215.420020]: cleared current plan +[INFO] [1735961215.420321]: moving to home +[INFO] [1735961219.125216]: reached home +[INFO] [1735961219.136572]: reset action history +[INFO] [1735961222.000706]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-metadata.json b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..19cd249d7adc090e7a560cfa09bfbdd99f6e2c18 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:23:09.694174", + "startedAt": "2025-01-04T03:23:09.354185", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4320.156875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4517.955, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4510.961, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 838.2302551269531 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7b15f49e59c8bc87fdd15eeec476012c32b376c7 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "204e5f6d515db78c9797f141a381290f375d6efe7941034acf7efa403cedd34b", "size": 1138629, "path": "media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4"}, "global_step": 665, "_timestamp": 1735961235.797489, "_runtime": 246.43412804603577, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b39b06e75cb8443c40e412577a5428c4a526c986e6499a341b7bc0f1565e98fc", "size": 1133959, "path": "media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "25b75c49234e97b158f8f0c9df591cd99b39e05b03a836b1e628f578caf03c4c", "size": 1138515, "path": "media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "75933aeb4ef7e707523abcd74cbddf7203aa7422728f68ebceee9b7883b70d2d", "size": 1175309, "path": "media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "b7d77604e2e08eabb6333a7a8a0ea6f062a198b092e5f6e6eb38cba554f1d3ce", "size": 1138986, "path": "media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://dmydevpsun72o19lkx8xztlq9sjr7zf62dnf7hghmfrte98on1xghe86q038atpmi4w30o2r9uu97agmhvplf99jpm3xzif9qxukdr4ugfw7ilwv88bda11htyf7r4qv/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://nevlexrq15uw4twy6symgkyaf39ptulsbg4y6a3i7mybrlqnrtpc3lilz7k4nopfv349ns7zjjnvtfzsko1dmgpas6jlggiw73376y04i1i5wd5ac9ftd9p4lby1oly8:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 20, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9946215747797684, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8453640490079612, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.865594230519132, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c68d11fd8ad6b790b7a949bb003b7716c3f8e083e007b71762bd414593b8472d", "size": 419, "artifact_path": "wandb-client-artifact://ldfn3183vto1jn6f0a4yd8cclc4jggx5zo6idds6djfq5nfbwb1tszvhokcb61iph7jewht6eelycmpc5du16159bhb8xwqxme2gbfal8lgp8pazas6obyn4cclcyux5/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://3rgup2ejmir17k3u03d423vky15w8hhmqok8ovrxjzfcwlvuh5jvx4huul6kwl3ymcjcer87cgflaid4ur6e80iok29auoalmd70n5l50an2wip3pvhxdk71zdhm76r8:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 249}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9946215748786926, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8453640341758728, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.865594208240509, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug-internal.log b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e1c754418a7ffa1c6038f6143c9d2d5f8672bea8 --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug-internal.log @@ -0,0 +1,740 @@ +2025-01-03 22:23:09,363 INFO StreamThr :1982600 [internal.py:wandb_internal():85] W&B internal server running at pid: 1982600, started at: 2025-01-03 22:23:09.363010 +2025-01-03 22:23:09,364 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:23:09,375 INFO WriterThread:1982600 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/run-6mzw1wsh.wandb +2025-01-03 22:23:09,375 DEBUG SenderThread:1982600 [sender.py:send():379] send: header +2025-01-03 22:23:09,382 DEBUG SenderThread:1982600 [sender.py:send():379] send: run +2025-01-03 22:23:09,590 INFO SenderThread:1982600 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files +2025-01-03 22:23:09,590 INFO SenderThread:1982600 [sender.py:_start_run_threads():1188] run started: 6mzw1wsh with start time 1735960989.363361 +2025-01-03 22:23:09,606 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:23:09,606 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:23:09,676 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:23:09,681 DEBUG HandlerThread:1982600 [system_info.py:__init__():26] System info init +2025-01-03 22:23:09,681 DEBUG HandlerThread:1982600 [system_info.py:__init__():41] System info init done +2025-01-03 22:23:09,681 INFO HandlerThread:1982600 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:23:09,681 INFO SystemMonitor:1982600 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:23:09,681 INFO HandlerThread:1982600 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:23:09,682 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:23:09,682 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:23:09,683 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:23:09,683 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:23:09,684 INFO SystemMonitor:1982600 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:23:09,694 DEBUG HandlerThread:1982600 [system_info.py:probe():152] Probing system +2025-01-03 22:23:09,695 DEBUG HandlerThread:1982600 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_info.py:probe():200] Probing system done +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:23:09.694174', 'startedAt': '2025-01-04T03:23:09.354185', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4320.156875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4517.955, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4510.961, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 838.2302551269531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:23:09,700 INFO HandlerThread:1982600 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:23:09,700 INFO HandlerThread:1982600 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:23:09,700 DEBUG HandlerThread:1982600 [system_info.py:_save_code():44] Saving code +2025-01-03 22:23:09,701 DEBUG HandlerThread:1982600 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:23:09,701 DEBUG HandlerThread:1982600 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:23:09,764 DEBUG HandlerThread:1982600 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:23:09,764 INFO HandlerThread:1982600 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:23:09,767 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:23:09,767 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:23:09,767 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:23:09,768 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:23:10,000 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:23:10,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:23:10,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:10,002 DEBUG SenderThread:1982600 [sender.py:send():379] send: telemetry +2025-01-03 22:23:10,019 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/pw0qsnwy-wandb-metadata.json +2025-01-03 22:23:10,048 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:10,048 INFO HandlerThread:1982600 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/tb" + save: true +} +control { + relay_id: "e231d7849ef446b99386d81c71cff138" +} +_info { + stream_id: "6mzw1wsh" +} + +2025-01-03 22:23:10,049 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:10,050 DEBUG HandlerThread:1982600 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:23:10,067 INFO wandb-upload_2:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/1cvmaba9-diff.patch +2025-01-03 22:23:10,079 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/86ump4oz-code/FISH/eval_robot.py +2025-01-03 22:23:10,177 DEBUG SenderThread:1982600 [sender.py:send():379] send: tbrecord +2025-01-03 22:23:10,177 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:23:10,177 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 with policy live +2025-01-03 22:23:10,375 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/w56zewy7-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH/eval_robot.py +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-metadata.json +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/diff.patch +2025-01-03 22:23:10,592 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:10,593 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code +2025-01-03 22:23:10,593 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH +2025-01-03 22:23:11,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:12,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:12,591 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:13,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:14,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:15,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:15,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:16,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:17,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:18,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:19,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:20,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:20,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:21,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:22,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:22,375 DEBUG SenderThread:1982600 [sender.py:send():379] send: config +2025-01-03 22:23:23,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:24,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:24,594 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:25,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:25,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:25,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:25,093 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:26,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:27,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:28,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:28,595 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:29,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:30,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:31,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:31,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:32,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:32,596 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:33,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:34,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:35,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:36,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:36,473 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:36,597 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:37,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:38,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:38,598 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:39,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:40,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:40,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:40,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:40,598 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:23:41,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:42,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:42,086 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:42,599 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml +2025-01-03 22:23:43,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:44,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:45,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:46,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:47,008 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:48,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:48,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:49,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:50,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:51,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:52,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:53,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:54,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:23:54,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:55,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:23:55,001 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:23:55,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:56,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:57,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:58,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:59,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:23:59,121 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:00,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:01,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:02,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:03,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:04,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:05,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:05,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:06,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:06,605 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:07,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:07,185 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:24:07,186 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:24:07,186 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 with policy now +2025-01-03 22:24:07,186 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:24:07,186 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:24:07,187 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:24:07,456 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/rb74u4is-media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:24:07,605 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media +2025-01-03 22:24:07,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:24:08,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:08,027 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/6czpcucr-media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:24:08,606 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:09,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:09,684 DEBUG SystemMonitor:1982600 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:24:09,685 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:24:10,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:10,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:10,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:10,072 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:11,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:12,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:12,607 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:13,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:14,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:14,608 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:15,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:16,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:16,013 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:16,609 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:24:16,861 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/fyy195pm-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:24:17,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:18,013 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:18,610 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:19,013 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:20,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:21,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:21,250 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:22,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:22,611 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:23,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:24,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:25,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:25,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:25,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:26,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:27,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:27,084 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:28,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:29,014 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:30,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:31,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:32,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:33,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:33,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:34,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:35,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:36,015 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:37,016 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:38,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:38,016 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:39,018 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:39,685 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:24:40,001 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:40,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:40,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:41,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:42,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:43,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:43,088 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:44,020 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:45,021 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:46,021 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:47,022 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:48,022 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:48,279 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:48,361 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:24:48,361 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 with policy now +2025-01-03 22:24:48,363 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:24:48,364 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:24:48,364 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:24:48,364 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:24:48,619 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:48,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:24:48,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:24:48,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:24:48,627 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/pnzyxy9r-media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:24:48,881 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/bd9fft1d-media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:24:49,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:50,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:50,620 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:51,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:52,029 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:53,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:54,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:54,121 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:24:54,621 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:55,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:24:55,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:24:55,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:56,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:56,622 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:24:57,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:58,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:24:58,479 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:24:58,480 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:24:58,481 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:24:58,623 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:24:58,623 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:24:59,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:00,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:00,024 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:00,624 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:01,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:02,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:02,624 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:03,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:04,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:04,625 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:05,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:06,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:06,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:07,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:08,025 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:09,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:09,686 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:25:10,002 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:10,002 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:10,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:11,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:11,164 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:12,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:13,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:14,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:15,026 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:16,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:17,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:17,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:18,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:19,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:20,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:21,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:22,005 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:22,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:23,027 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:24,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:25,003 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:25,004 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:25,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:26,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:27,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:27,081 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:28,028 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:29,032 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:30,037 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:30,136 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:25:30,136 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 with policy now +2025-01-03 22:25:30,138 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:25:30,139 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:25:30,139 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:25:30,140 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:25:30,418 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/njth5p9o-media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:25:30,634 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:30,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:25:30,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:25:30,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:25:30,904 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/38q3h014-media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:25:31,035 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:32,036 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:32,162 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:32,635 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:33,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:34,043 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:35,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:36,044 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:36,637 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:37,037 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:37,649 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:38,039 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:38,637 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:39,038 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:39,686 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:25:40,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:40,005 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:40,039 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:40,308 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:25:40,309 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:25:40,310 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:25:40,638 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:40,638 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:25:40,638 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:25:40,897 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/0rjd4ust-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:25:41,049 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:42,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:42,639 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:42,885 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:43,044 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:44,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:44,639 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:45,053 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:46,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:46,640 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:25:47,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:48,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:48,054 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:49,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:50,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:51,040 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:52,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:53,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:54,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:25:54,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:55,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:25:55,004 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:25:55,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:56,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:57,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:58,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:59,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:25:59,072 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:00,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:01,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:02,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:03,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:04,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:04,401 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:04,646 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:05,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:06,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:07,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:08,042 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:08,647 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:09,043 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:09,687 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:26:09,687 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:10,004 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:10,004 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:10,045 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:11,043 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:12,046 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:12,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:13,046 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:13,053 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:26:13,053 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 with policy now +2025-01-03 22:26:13,097 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:26:13,098 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:26:13,098 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:26:13,098 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:26:13,327 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/7zcazti4-media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:26:13,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:26:13,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:26:13,650 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:26:13,917 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/ef69g7vh-media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:26:14,049 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:14,649 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:15,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:15,048 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:16,051 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:17,056 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:18,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:18,651 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:19,061 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:20,048 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:20,651 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:21,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:21,051 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:22,051 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:23,053 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:23,652 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:26:24,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:24,363 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:26:24,363 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:26:24,364 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:26:24,653 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:24,653 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:26:25,007 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:25,007 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:25,060 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:26,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:26,117 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:26,654 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:27,059 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:28,054 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:28,654 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:29,054 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:30,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:31,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:32,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:32,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:33,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:34,055 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:35,056 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:36,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:37,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:37,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:38,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:39,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:39,687 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:26:40,006 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:40,006 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:40,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:41,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:42,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:42,122 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:43,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:44,057 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:45,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:46,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:47,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:48,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:48,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:49,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:50,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:51,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:52,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:53,010 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:53,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:54,058 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:55,011 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:26:55,012 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:26:55,067 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:55,405 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:26:55,405 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 with policy now +2025-01-03 22:26:55,407 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:26:55,407 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:26:55,407 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:26:55,410 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:26:55,656 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/958ilpfd-media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:26:55,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:26:55,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:26:55,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval +2025-01-03 22:26:55,932 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/qysku1ms-media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:26:56,061 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:56,664 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:26:57,065 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:58,068 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:26:58,421 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:26:59,069 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:00,073 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:00,665 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:01,072 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:02,070 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:03,075 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:04,041 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:04,075 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:04,667 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:05,071 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:06,073 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:07,081 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:08,079 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:09,078 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:09,687 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:27:09,688 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:10,009 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:27:10,010 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:27:10,079 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:11,080 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:12,080 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:13,080 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:14,090 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:15,012 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:15,093 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:15,389 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:27:15,390 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:27:15,539 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table +2025-01-03 22:27:15,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval +2025-01-03 22:27:15,860 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/oacesd5o-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:15,949 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/o2zlq7tq-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:16,041 INFO wandb-upload_2:1982600 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpiya1qizf +2025-01-03 22:27:16,077 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:16,454 INFO SenderThread:1982600 [sender.py:send_request_log_artifact():1525] logged artifact run-6mzw1wsh-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3MzU2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4Nzg0OQ==', 'latestArtifact': None}} +2025-01-03 22:27:16,454 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:16,454 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,455 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,455 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:27:16,455 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-03 22:27:16,455 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:16,455 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,459 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,459 DEBUG SenderThread:1982600 [sender.py:send():379] send: telemetry +2025-01-03 22:27:16,459 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,460 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,460 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,460 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,460 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,460 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,461 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,461 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,461 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,461 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,462 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,462 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,462 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,462 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,462 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,463 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,463 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,463 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,463 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,464 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,464 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:16,464 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:16,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:16,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:16,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:16,713 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/lho5mtpi-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:17,078 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:17,370 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:27:17,370 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:27:17,413 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:27:17,671 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:17,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval +2025-01-03 22:27:17,752 INFO wandb-upload_2:1982600 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptu9wmfj3 +2025-01-03 22:27:17,856 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/9jvdau5e-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:18,078 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:18,261 INFO SenderThread:1982600 [sender.py:send_request_log_artifact():1525] logged artifact run-6mzw1wsh-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3MzYwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4Nzg1Ng==', 'latestArtifact': None}} +2025-01-03 22:27:18,261 DEBUG SenderThread:1982600 [sender.py:send():379] send: files +2025-01-03 22:27:18,261 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json with policy now +2025-01-03 22:27:18,262 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:18,262 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:18,262 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:18,498 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/gdm6fide-media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:18,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:18,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:18,877 DEBUG SenderThread:1982600 [sender.py:send():379] send: exit +2025-01-03 22:27:18,877 INFO SenderThread:1982600 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:27:18,877 INFO SenderThread:1982600 [sender.py:send_exit():588] handling runtime: 249 +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:send_exit():594] send defer +2025-01-03 22:27:18,878 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,878 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:27:18,878 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:27:18,878 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:27:18,878 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,879 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:27:18,879 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,879 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:27:18,879 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:27:18,879 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,879 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:27:18,879 INFO HandlerThread:1982600 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:27:18,879 DEBUG SystemMonitor:1982600 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:27:18,879 DEBUG SystemMonitor:1982600 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:27:18,880 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:27:18,880 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:27:18,934 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:27:18,934 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:27:18,934 INFO HandlerThread:1982600 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:27:18,934 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,934 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:27:18,934 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:27:18,934 DEBUG SenderThread:1982600 [sender.py:send():379] send: stats +2025-01-03 22:27:18,935 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,935 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:27:18,935 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:18,935 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:27:18,935 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:27:18,935 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:18,935 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:27:19,672 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:20,673 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:25,435 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:25,435 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:25,435 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:27:25,436 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,436 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:27:25,436 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:27:25,436 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:25,436 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:25,437 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:27:25,437 DEBUG SenderThread:1982600 [sender.py:send():379] send: history +2025-01-03 22:27:25,437 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:27:25,438 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:25,438 DEBUG SenderThread:1982600 [sender.py:send():379] send: summary +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:27:25,442 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:27:25,442 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:25,442 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:27:25,442 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,442 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:27:25,446 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:27:25,572 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:27:25,572 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:25,572 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:27:25,572 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:25,572 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:27:25,675 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml +2025-01-03 22:27:25,675 INFO Thread-12 :1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:25,878 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:28,439 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:27:28,439 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:28,440 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:28,440 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:27:28,440 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:28,440 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:27:28,440 INFO SenderThread:1982600 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:27:28,442 INFO SenderThread:1982600 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:27:28,444 INFO SenderThread:1982600 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:27:28,456 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:27:28,456 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:28,457 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:27:28,457 DEBUG SenderThread:1982600 [sender.py:send():379] send: artifact +2025-01-03 22:27:28,663 INFO SenderThread:1982600 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:27:28,663 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:28,664 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:27:28,664 INFO SenderThread:1982600 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:27:28,676 INFO SenderThread:1982600 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:28,676 INFO SenderThread:1982600 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml config.yaml +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json wandb-summary.json +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt requirements.txt +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log output.log +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/diff.patch diff.patch +2025-01-03 22:27:28,677 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 media/videos/eval/2_eval_2_25b75c49234e97b158f8.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 media/videos/eval/0_eval_0_204e5f6d515db78c9797.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 media/videos/eval/1_eval_1_b39b06e75cb8443c40e4.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 media/videos/eval/3_eval_3_75933aeb4ef7e707523a.mp4 +2025-01-03 22:27:28,681 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 media/videos/eval/4_eval_4_b7d77604e2e08eabb633.mp4 +2025-01-03 22:27:28,684 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json media/table/eval/total_success_rate_ci_6_c68d11fd8ad6b790b7a9.table.json +2025-01-03 22:27:28,684 INFO SenderThread:1982600 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:27:28,685 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:27:28,685 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:28,686 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:27:28,686 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:28,686 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:27:28,686 INFO SenderThread:1982600 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:27:28,865 INFO wandb-upload_2:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/wandb-summary.json +2025-01-03 22:27:28,876 INFO wandb-upload_0:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/config.yaml +2025-01-03 22:27:28,880 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:28,880 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:28,896 INFO wandb-upload_4:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/output.log +2025-01-03 22:27:28,901 INFO wandb-upload_3:1982600 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/files/requirements.txt +2025-01-03 22:27:28,993 INFO wandb-upload_1:1982600 [upload_job.py:push():130] Uploaded file /tmp/tmpoc_i5w0twandb/fb5fzend-events.out.tfevents.1735960990.leonmkim-ROG-Strix-G15CS-G15CS.1982433.0 +2025-01-03 22:27:29,194 INFO Thread-11 (_thread_body):1982600 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:27:29,194 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,194 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:27:29,194 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,194 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:27:29,194 INFO SenderThread:1982600 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:27:29,197 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:27:29,198 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,198 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:27:29,198 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,198 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:27:29,198 INFO SenderThread:1982600 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:27:29,391 INFO SenderThread:1982600 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:27:29,392 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:27:29,392 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,392 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:27:29,392 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,392 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:27:29,392 INFO SenderThread:1982600 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:27:29,392 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:27:29,393 INFO HandlerThread:1982600 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:27:29,393 DEBUG SenderThread:1982600 [sender.py:send():379] send: final +2025-01-03 22:27:29,393 DEBUG SenderThread:1982600 [sender.py:send():379] send: footer +2025-01-03 22:27:29,393 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: defer +2025-01-03 22:27:29,393 INFO SenderThread:1982600 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:27:29,393 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:29,394 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:27:29,394 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:27:29,394 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:27:29,395 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:27:29,395 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:29,395 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:27:29,395 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:27:29,450 DEBUG SenderThread:1982600 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:27:29,450 INFO MainThread:1982600 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:27:29,450 INFO MainThread:1982600 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:27:29,451 INFO MainThread:1982600 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:27:29,451 DEBUG HandlerThread:1982600 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:27:29,451 INFO HandlerThread:1982600 [handler.py:finish():882] shutting down handler +2025-01-03 22:27:30,394 INFO WriterThread:1982600 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/run-6mzw1wsh.wandb +2025-01-03 22:27:30,450 INFO SenderThread:1982600 [sender.py:finish():1615] shutting down sender +2025-01-03 22:27:30,450 INFO SenderThread:1982600 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:27:30,450 INFO SenderThread:1982600 [file_pusher.py:join():175] waiting for file pusher diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug.log b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2221bf96552f9d18ff08fb796a8da1404bd0be9b --- /dev/null +++ b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Configure stats pid to 1982433 +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:23:09,355 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/settings +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug.log +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/logs/debug-internal.log +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh'} +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():619] starting backend +2025-01-03 22:23:09,356 INFO MainThread:1982433 [wandb_init.py:init():623] setting up manager +2025-01-03 22:23:09,362 INFO MainThread:1982433 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:23:09,363 INFO MainThread:1982433 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:23:09,374 INFO MainThread:1982433 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:23:09,380 INFO MainThread:1982433 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:23:09,605 INFO MainThread:1982433 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:23:09,671 INFO MainThread:1982433 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:23:09,671 INFO MainThread:1982433 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:23:10,001 INFO MainThread:1982433 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:23:10,003 INFO MainThread:1982433 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:23:10,004 INFO MainThread:1982433 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/6mzw1wsh/tb, True +2025-01-03 22:23:22,375 INFO MainThread:1982433 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:27:30,452 WARNING MsgRouterThr:1982433 [router.py:message_loop():77] message_loop has been closed diff --git a/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/run-6mzw1wsh.wandb b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/run-6mzw1wsh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c63184069a3cf6da8bae30c268fc7a599acaeda3 Binary files /dev/null and b/6mzw1wsh/wandb/run-20250103_222309-6mzw1wsh/run-6mzw1wsh.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73a45f77980a87a653a0fe48fce04efcdd11f479 --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 1 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1922_1' +wandb_notes: 1922_1_restarted_3 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/l2vooc6k/.hydra/config.yaml b/l2vooc6k/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/l2vooc6k/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/l2vooc6k/.hydra/hydra.yaml b/l2vooc6k/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9638b2c0398cb9d1a4d90f399b677f831c4cfadd --- /dev/null +++ b/l2vooc6k/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/l2vooc6k/.hydra/overrides.yaml b/l2vooc6k/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/l2vooc6k/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/l2vooc6k/episode_rosbags/aligned_depth_to_color_K.npy b/l2vooc6k/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/l2vooc6k/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/l2vooc6k/episode_rosbags/cam_tf_world.npy b/l2vooc6k/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/l2vooc6k/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/l2vooc6k/episode_rosbags/color_K.npy b/l2vooc6k/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/l2vooc6k/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/l2vooc6k/episode_rosbags/depth_K.npy b/l2vooc6k/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/l2vooc6k/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/l2vooc6k/eval_robot.log b/l2vooc6k/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..7d5d36b123cbe863348e841be0cc263b83d5835c --- /dev/null +++ b/l2vooc6k/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 21:02:50,537][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:02:50,540][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/l2vooc6k/eval_video/0_eval.mp4 b/l2vooc6k/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b47c12fe5d25d8804e672a9f212383caf16240b --- /dev/null +++ b/l2vooc6k/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c6c007478e4fec430512ffde6b8aec6b9463b1fcdf04c4b085f1c37417e180 +size 1155189 diff --git a/l2vooc6k/eval_video/1_eval.mp4 b/l2vooc6k/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d877e34e164ce58c5b35b82a90780e2ebd2688a7 --- /dev/null +++ b/l2vooc6k/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdfd18c4a1782d10be6a08ed84572c8662c780962ee66acd115847f54437d391 +size 1156643 diff --git a/l2vooc6k/eval_video/2_eval.mp4 b/l2vooc6k/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d7529b03e076acab330bc2f7eed9e2a890b44a96 --- /dev/null +++ b/l2vooc6k/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf28e457796d51f7468174bc4b716fb4bdeb422c76f450a360ef4ba6308a13d4 +size 1160912 diff --git a/l2vooc6k/eval_video/3_eval.mp4 b/l2vooc6k/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f2be6433b0eadaced70c0a7819e86a8ea46d198a --- /dev/null +++ b/l2vooc6k/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05a6da1b8b18548e98dc1b3ef6612140bfd2f89cde0d9200160c31bd04c78166 +size 1153377 diff --git a/l2vooc6k/eval_video/4_eval.mp4 b/l2vooc6k/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33006682f59a5e0d75b86ce2cb992bb913be94fa --- /dev/null +++ b/l2vooc6k/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd22cdf48d22cf5a74051cd94e780009189a307951996230c198f9d7b9c5c0a +size 1208016 diff --git a/l2vooc6k/tb/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 b/l2vooc6k/tb/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 new file mode 100644 index 0000000000000000000000000000000000000000..240dc4551405de153bccb827a8ffa6879b1b72f4 --- /dev/null +++ b/l2vooc6k/tb/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:745a1d7cb5a4ec1cb9a6caaedead626661a1aa57e3b370c18f1fa85049c1942f +size 6539 diff --git a/l2vooc6k/wandb/debug-internal.log b/l2vooc6k/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..87dee0c68e88ac4c79084688a3fdabcf4e6efabe --- /dev/null +++ b/l2vooc6k/wandb/debug-internal.log @@ -0,0 +1,786 @@ +2025-01-04 21:02:56,374 INFO StreamThr :2021275 [internal.py:wandb_internal():85] W&B internal server running at pid: 2021275, started at: 2025-01-04 21:02:56.374648 +2025-01-04 21:02:56,376 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:02:56,386 INFO WriterThread:2021275 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/run-l2vooc6k.wandb +2025-01-04 21:02:56,387 DEBUG SenderThread:2021275 [sender.py:send():379] send: header +2025-01-04 21:02:56,395 DEBUG SenderThread:2021275 [sender.py:send():379] send: run +2025-01-04 21:02:56,642 INFO SenderThread:2021275 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files +2025-01-04 21:02:56,642 INFO SenderThread:2021275 [sender.py:_start_run_threads():1188] run started: l2vooc6k with start time 1736042576.374716 +2025-01-04 21:02:56,659 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:02:56,659 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:02:56,729 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:02:56,734 DEBUG HandlerThread:2021275 [system_info.py:__init__():26] System info init +2025-01-04 21:02:56,734 DEBUG HandlerThread:2021275 [system_info.py:__init__():41] System info init done +2025-01-04 21:02:56,734 INFO HandlerThread:2021275 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:02:56,734 INFO SystemMonitor:2021275 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:02:56,735 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:02:56,735 INFO HandlerThread:2021275 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:02:56,735 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:02:56,736 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:02:56,737 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:02:56,737 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:02:56,747 DEBUG HandlerThread:2021275 [system_info.py:probe():152] Probing system +2025-01-04 21:02:56,749 DEBUG HandlerThread:2021275 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_info.py:probe():200] Probing system done +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:02:56.747830', 'startedAt': '2025-01-05T02:02:56.363221', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3941.48725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.933, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.323, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 643.6809577941895}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:02:56,754 INFO HandlerThread:2021275 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:02:56,754 INFO HandlerThread:2021275 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_info.py:_save_code():44] Saving code +2025-01-04 21:02:56,756 DEBUG HandlerThread:2021275 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:02:56,756 DEBUG HandlerThread:2021275 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:02:56,818 DEBUG HandlerThread:2021275 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:02:56,818 INFO HandlerThread:2021275 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:02:56,822 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:02:56,822 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:02:56,822 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:02:56,822 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:02:57,074 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/p24k1bf2-wandb-metadata.json +2025-01-04 21:02:57,082 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:02:57,082 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:02:57,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:02:57,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:02:57,095 INFO wandb-upload_2:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/puvneh0w-diff.patch +2025-01-04 21:02:57,117 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/j71v2x6k-code/FISH/eval_robot.py +2025-01-04 21:02:57,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:57,125 INFO HandlerThread:2021275 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/tb" + save: true +} +control { + relay_id: "34850869c6cf4c14b3fc83ae75ff0b9f" +} +_info { + stream_id: "l2vooc6k" +} + +2025-01-04 21:02:57,125 DEBUG HandlerThread:2021275 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:02:57,236 DEBUG SenderThread:2021275 [sender.py:send():379] send: telemetry +2025-01-04 21:02:57,237 DEBUG SenderThread:2021275 [sender.py:send():379] send: tbrecord +2025-01-04 21:02:57,237 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:02:57,237 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 with policy live +2025-01-04 21:02:57,455 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/dde2c3ad-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/diff.patch +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-metadata.json +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH/eval_robot.py +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:02:57,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:02:57,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code +2025-01-04 21:02:57,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH +2025-01-04 21:02:58,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:59,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:59,643 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:00,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:01,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:02,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:02,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:02,662 DEBUG SenderThread:2021275 [sender.py:send():379] send: config +2025-01-04 21:03:03,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:03,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:04,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:05,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:05,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:06,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:07,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:07,386 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:07,646 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:08,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:09,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:09,646 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:10,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:11,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:11,647 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:12,082 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:12,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:12,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:13,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:13,151 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:14,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:15,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:16,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:17,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:18,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:19,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:19,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:20,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:21,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:22,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:23,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:24,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:24,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:25,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:26,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:27,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:27,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:27,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:28,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:29,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:29,245 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:29,652 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml +2025-01-04 21:03:30,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:31,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:32,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:33,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:34,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:35,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:35,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:36,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:37,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:38,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:39,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:40,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:40,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:41,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:42,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:42,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:42,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:43,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:44,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:45,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:45,182 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:46,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:47,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:48,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:49,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:50,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:51,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:51,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:52,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:53,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:54,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:55,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:56,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:56,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:56,737 DEBUG SystemMonitor:2021275 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:03:56,738 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:03:57,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:57,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:57,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:57,660 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:58,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:59,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:59,660 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:00,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:01,091 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:01,661 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:01,831 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:02,094 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:03,094 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:03,661 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:04,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:05,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:06,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:07,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:07,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:08,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:09,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:10,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:11,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:12,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:12,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:12,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:12,142 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:13,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:14,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:15,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:16,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:17,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:18,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:18,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:19,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:20,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:21,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:22,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:23,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:24,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:24,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:25,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:26,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:26,738 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:04:27,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:27,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:27,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:28,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:28,490 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:04:28,490 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 with policy now +2025-01-04 21:04:28,490 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:04:28,491 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:04:28,491 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:04:28,492 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:04:28,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:04:28,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:04:28,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:04:28,669 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos +2025-01-04 21:04:28,669 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media +2025-01-04 21:04:28,757 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/mkpq3ptv-media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:04:29,012 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/ny3dotta-media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:04:29,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:29,502 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:29,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:30,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:31,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:32,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:33,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:33,669 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:34,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:34,919 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:35,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:36,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:37,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:37,672 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:38,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:39,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:39,673 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:39,673 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:04:39,914 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/o6fomjur-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:04:40,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:40,885 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:41,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:41,673 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:42,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:42,084 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:42,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:43,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:43,674 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:44,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:45,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:46,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:46,830 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:47,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:48,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:49,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:50,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:51,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:52,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:52,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:53,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:54,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:55,120 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:56,120 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:56,739 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:04:57,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:57,085 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:57,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:57,173 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:57,678 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:58,120 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:59,122 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:00,122 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:01,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:02,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:03,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:03,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:04,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:05,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:06,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:07,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:08,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:08,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:09,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:10,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:10,699 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:05:10,699 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 with policy now +2025-01-04 21:05:10,699 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:05:10,700 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:05:10,700 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:05:10,700 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:05:10,966 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/prn4p2zm-media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:05:11,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:11,682 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:05:11,683 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:11,683 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:05:11,683 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:05:11,974 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/zajpxdfm-media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:05:12,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:12,086 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:12,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:13,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:13,190 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:14,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:15,134 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:15,684 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:16,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:17,128 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:18,127 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:18,245 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:19,127 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:19,685 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:20,128 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:20,586 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:05:20,586 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:05:20,587 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:05:20,685 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:05:20,686 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:05:21,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:21,686 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:22,128 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:23,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:23,609 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:23,687 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:24,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:25,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:25,687 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:26,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:26,739 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:05:27,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:27,084 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:27,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:28,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:29,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:29,231 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:30,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:31,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:32,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:33,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:34,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:35,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:35,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:36,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:37,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:38,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:39,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:40,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:40,132 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:41,132 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:42,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:42,084 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:42,132 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:43,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:44,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:45,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:45,161 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:46,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:47,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:48,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:49,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:50,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:50,604 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:51,135 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:51,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:51,978 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:05:51,978 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 with policy now +2025-01-04 21:05:51,981 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:05:51,982 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:05:51,982 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:05:51,982 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:05:52,134 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:52,240 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/4v6ig217-media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:05:52,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:05:52,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:05:52,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:05:52,982 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/vmr8xpxz-media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:05:53,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:53,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:54,153 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:54,697 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:55,140 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:55,705 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:56,141 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:56,740 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:05:57,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:57,090 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:57,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:57,698 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:58,146 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:59,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:59,698 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:00,146 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:00,900 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:01,142 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:01,400 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:06:01,401 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:06:01,401 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:06:01,699 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:06:01,699 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:01,699 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:06:01,948 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/shnt39x8-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:06:02,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:03,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:03,700 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:04,150 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:05,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:05,701 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:05,941 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:06,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:07,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:08,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:09,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:10,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:11,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:11,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:12,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:12,085 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:12,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:13,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:14,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:15,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:16,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:16,173 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:17,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:18,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:19,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:20,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:21,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:22,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:22,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:23,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:24,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:25,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:26,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:26,740 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:06:27,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:27,086 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:27,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:27,198 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:28,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:29,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:30,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:31,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:32,172 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:32,424 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:32,752 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:06:32,753 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 with policy now +2025-01-04 21:06:32,762 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:06:32,763 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:06:32,763 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:06:32,763 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:06:33,016 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/efsavywt-media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:06:33,150 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:06:34,033 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/705cstlg-media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:06:34,153 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:35,158 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:36,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:37,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:37,712 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:38,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:38,170 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:39,156 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:40,161 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:41,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:41,714 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:42,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:42,095 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:42,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:43,160 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:43,196 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:44,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:45,167 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:46,162 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:47,162 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:48,164 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:49,098 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:49,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:49,716 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:06:50,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:50,458 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:06:50,458 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:06:50,459 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:06:50,717 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:06:51,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:51,717 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:52,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:53,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:54,174 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:54,341 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:55,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:55,718 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:56,167 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:56,741 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:06:57,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:57,087 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:57,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:57,719 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:58,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:59,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:00,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:00,228 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:01,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:02,169 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:03,169 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:04,169 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:05,170 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:05,265 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:05,722 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:06,174 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:07,174 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:08,177 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:09,177 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:10,178 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:11,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:11,178 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:12,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:07:12,087 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:07:12,178 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:13,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:14,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:15,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:16,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:16,189 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:17,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:18,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:19,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:19,727 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:20,180 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:21,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:21,501 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:22,191 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:22,682 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:07:22,682 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 with policy now +2025-01-04 21:07:22,685 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:07:22,685 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:22,685 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:22,685 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:22,728 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:22,728 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:22,729 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:07:22,946 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/ojvaljad-media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:23,199 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:23,215 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/joaa1t3v-media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:23,728 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:24,202 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:25,196 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:26,200 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:26,741 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:07:26,741 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:27,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:07:27,096 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:07:27,203 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:27,730 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:28,200 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:29,216 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:30,204 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:31,199 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:31,731 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:32,099 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:32,203 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:33,205 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:34,215 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:35,208 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:36,218 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:37,218 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:37,481 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:07:37,481 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:07:37,700 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:07:37,733 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:37,733 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval +2025-01-04 21:07:37,922 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/q79izaar-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:37,992 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/qcg0dzhm-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:38,022 INFO wandb-upload_2:2021275 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprehdr76g +2025-01-04 21:07:38,116 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:07:38,127 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:07:38,216 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:38,394 INFO SenderThread:2021275 [sender.py:send_request_log_artifact():1525] logged artifact run-l2vooc6k-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzNTY5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzAwNw==', 'latestArtifact': None}} +2025-01-04 21:07:38,395 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:38,395 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,395 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:38,395 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:07:38,396 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,396 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send():379] send: telemetry +2025-01-04 21:07:38,397 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,397 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,397 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,397 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,398 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,398 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,398 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,398 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,399 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,400 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,401 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,401 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,401 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,401 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,402 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,402 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,402 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,402 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,402 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,403 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,403 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,403 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,403 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:07:38,634 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/b1yjdlo0-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval +2025-01-04 21:07:38,794 INFO wandb-upload_2:2021275 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfjmsyvm8 +2025-01-04 21:07:38,925 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/13v7wzvc-media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:39,162 INFO SenderThread:2021275 [sender.py:send_request_log_artifact():1525] logged artifact run-l2vooc6k-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzNTcxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzAxMg==', 'latestArtifact': None}} +2025-01-04 21:07:39,162 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:07:39,162 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json with policy now +2025-01-04 21:07:39,162 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:39,163 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:39,163 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:39,163 DEBUG SenderThread:2021275 [sender.py:send():379] send: exit +2025-01-04 21:07:39,163 INFO SenderThread:2021275 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:07:39,164 INFO SenderThread:2021275 [sender.py:send_exit():588] handling runtime: 281 +2025-01-04 21:07:39,167 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:send_exit():594] send defer +2025-01-04 21:07:39,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:07:39,168 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:07:39,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:07:39,168 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:07:39,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:07:39,169 DEBUG SystemMonitor:2021275 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:07:39,169 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:07:39,169 DEBUG SystemMonitor:2021275 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:07:39,169 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:07:39,222 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:07:39,222 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:07:39,222 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:07:39,223 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,223 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:07:39,223 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:07:39,223 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:07:39,223 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,223 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:07:39,223 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,223 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:07:39,224 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:07:39,224 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,224 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:07:39,398 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/x1rf1xvc-media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:39,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:39,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:41,735 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:45,521 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:45,521 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:45,521 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,522 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:07:45,522 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:45,523 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:45,523 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:07:45,523 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:45,524 DEBUG SenderThread:2021275 [sender.py:send():379] send: summary +2025-01-04 21:07:45,527 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:45,527 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,527 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:07:45,527 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:07:45,527 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:45,528 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:07:45,528 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,528 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:07:45,531 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:45,666 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:07:45,667 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:45,667 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:07:45,667 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,667 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:07:45,737 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:45,737 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml +2025-01-04 21:07:46,515 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:48,045 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:07:48,045 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:48,045 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:48,046 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:07:48,046 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:48,046 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:07:48,046 INFO SenderThread:2021275 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:07:48,048 INFO SenderThread:2021275 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:07:48,050 INFO SenderThread:2021275 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:07:48,064 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:07:48,065 DEBUG SenderThread:2021275 [sender.py:send():379] send: artifact +2025-01-04 21:07:48,067 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:48,068 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:07:48,237 INFO SenderThread:2021275 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:07:48,237 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:48,237 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:07:48,237 INFO SenderThread:2021275 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:07:48,515 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml config.yaml +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json wandb-summary.json +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt requirements.txt +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:07:48,739 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log output.log +2025-01-04 21:07:48,739 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/diff.patch diff.patch +2025-01-04 21:07:48,743 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:07:48,743 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:07:48,743 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:07:48,744 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:48,744 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:48,744 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:07:48,749 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:48,749 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:07:48,749 INFO SenderThread:2021275 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:07:48,928 INFO wandb-upload_2:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:48,949 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml +2025-01-04 21:07:48,952 INFO wandb-upload_3:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt +2025-01-04 21:07:48,984 INFO wandb-upload_4:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:49,018 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/9hue959f-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:49,219 INFO Thread-11 (_thread_body):2021275 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:07:49,219 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,219 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:07:49,219 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,219 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:07:49,220 INFO SenderThread:2021275 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:07:49,221 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:07:49,221 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,221 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:07:49,221 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,221 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:07:49,221 INFO SenderThread:2021275 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:07:49,446 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,446 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:07:49,446 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:07:49,446 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,446 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:07:49,447 DEBUG SenderThread:2021275 [sender.py:send():379] send: final +2025-01-04 21:07:49,447 DEBUG SenderThread:2021275 [sender.py:send():379] send: footer +2025-01-04 21:07:49,447 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,447 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:07:49,449 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:07:49,449 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:49,450 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:07:49,451 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:49,498 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:49,498 INFO MainThread:2021275 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:07:49,498 INFO MainThread:2021275 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:07:49,498 INFO MainThread:2021275 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:07:49,499 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:07:49,499 INFO HandlerThread:2021275 [handler.py:finish():882] shutting down handler +2025-01-04 21:07:50,449 INFO WriterThread:2021275 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/run-l2vooc6k.wandb +2025-01-04 21:07:50,498 INFO SenderThread:2021275 [sender.py:finish():1615] shutting down sender +2025-01-04 21:07:50,498 INFO SenderThread:2021275 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:07:50,498 INFO SenderThread:2021275 [file_pusher.py:join():175] waiting for file pusher diff --git a/l2vooc6k/wandb/debug.log b/l2vooc6k/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7cb5e7fcd4274fc0f9d27cc287dedc1e3be711e2 --- /dev/null +++ b/l2vooc6k/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Configure stats pid to 2021110 +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/settings +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug.log +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug-internal.log +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k'} +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():619] starting backend +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():623] setting up manager +2025-01-04 21:02:56,373 INFO MainThread:2021110 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:02:56,374 INFO MainThread:2021110 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:02:56,386 INFO MainThread:2021110 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:02:56,393 INFO MainThread:2021110 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:02:56,659 INFO MainThread:2021110 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:02:56,724 INFO MainThread:2021110 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:02:56,725 INFO MainThread:2021110 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:02:57,082 INFO MainThread:2021110 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:02:57,083 INFO MainThread:2021110 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:02:57,083 INFO MainThread:2021110 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:02:57,083 INFO MainThread:2021110 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:02:57,085 INFO MainThread:2021110 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:02:57,085 INFO MainThread:2021110 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/tb, True +2025-01-04 21:03:02,662 INFO MainThread:2021110 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 21:07:50,499 WARNING MsgRouterThr:2021110 [router.py:message_loop():77] message_loop has been closed diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH/eval_robot.py b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db95f9e85c8bd4b9173614404a37fcf14c81358c --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736042576 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/diff.patch b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..900e56acfbd743c95af462777b6882436e554ad3 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[33, 40, 0.825, 0.6728713810098137, 0.9291585086917867, 0.6784497797132267, 0.9146590530043026, 0.6985354423125878, 0.9084032847660557, 0.7483667879799973, 0.8952674660041156]]} \ No newline at end of file diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b47c12fe5d25d8804e672a9f212383caf16240b --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c6c007478e4fec430512ffde6b8aec6b9463b1fcdf04c4b085f1c37417e180 +size 1155189 diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d877e34e164ce58c5b35b82a90780e2ebd2688a7 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdfd18c4a1782d10be6a08ed84572c8662c780962ee66acd115847f54437d391 +size 1156643 diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d7529b03e076acab330bc2f7eed9e2a890b44a96 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf28e457796d51f7468174bc4b716fb4bdeb422c76f450a360ef4ba6308a13d4 +size 1160912 diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f2be6433b0eadaced70c0a7819e86a8ea46d198a --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05a6da1b8b18548e98dc1b3ef6612140bfd2f89cde0d9200160c31bd04c78166 +size 1153377 diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33006682f59a5e0d75b86ce2cb992bb913be94fa --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd22cdf48d22cf5a74051cd94e780009189a307951996230c198f9d7b9c5c0a +size 1208016 diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ce46fe3939079c74ac479e72a8f775116fd7e610 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736042582.662846]: resetting environment +[INFO] [1736042582.665519]: cleared current plan +[INFO] [1736042582.666388]: moving to home +[INFO] [1736042583.668525]: reached home +[INFO] [1736042583.669679]: reset action history +[INFO] [1736042586.378024]: environment reset +Starting episode 0 +[INFO] [1736042586.379951]: resetting environment +[INFO] [1736042586.382794]: cleared current plan +[INFO] [1736042586.383806]: moving to home +[INFO] [1736042587.385899]: reached home +[INFO] [1736042587.387328]: reset action history +[INFO] [1736042590.094424]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736042636.757819]: resetting environment +[INFO] [1736042636.760794]: cleared current plan +[INFO] [1736042636.761813]: moving to home +[INFO] [1736042637.764333]: reached home +[INFO] [1736042637.765970]: reset action history +[INFO] [1736042640.524219]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/episode_rosbags/episode_0_2025-01-04-21-04-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2021110.log +started roslaunch server http://158.130.50.37:38609/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2021389] +process[print_text-2]: started with pid [2021413] +process[rosbag_record-3]: started with pid [2021414] +started bagging! +[INFO] [1736042667.535366]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736042668.265811]: Stored episode 1. +[INFO] [1736042668.266550]: Saving video... +[INFO] [1736042668.467539]: Video saved! +Starting episode 1 +[INFO] [1736042668.492342]: resetting environment +[INFO] [1736042668.498710]: cleared current plan +[INFO] [1736042668.499857]: moving to home +[INFO] [1736042672.205274]: reached home +[INFO] [1736042672.205723]: reset action history +[INFO] [1736042674.919170]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042678.840883]: resetting environment +[INFO] [1736042678.842967]: cleared current plan +[INFO] [1736042678.843118]: moving to home +[INFO] [1736042679.844532]: reached home +[INFO] [1736042679.844987]: reset action history +[INFO] [1736042682.549854]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/episode_rosbags/episode_1_2025-01-04-21-04-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2021110.log +started roslaunch server http://158.130.50.37:38799/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2021535] +process[print_text-5]: started with pid [2021536] +process[rosbag_record-6]: started with pid [2021560] +started bagging! +For topic gripper_width: timestamp difference is -214389896 for nearest: 1736042695251508856 - target: 1736042695465898752 at idx 173 +[INFO] [1736042709.536589]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736042710.305420]: Stored episode 2. +[INFO] [1736042710.305692]: Saving video... +[INFO] [1736042710.664322]: Video saved! +Starting episode 2 +[INFO] [1736042710.705117]: resetting environment +[INFO] [1736042710.715500]: cleared current plan +[INFO] [1736042710.716535]: moving to home +[INFO] [1736042714.421181]: reached home +[INFO] [1736042714.427429]: reset action history +[INFO] [1736042717.201325]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042719.889157]: resetting environment +[INFO] [1736042719.897387]: cleared current plan +[INFO] [1736042719.897679]: moving to home +[INFO] [1736042720.900423]: reached home +[INFO] [1736042720.900946]: reset action history +[INFO] [1736042723.608505]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/episode_rosbags/episode_2_2025-01-04-21-05-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2021110.log +started roslaunch server http://158.130.50.37:42635/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2021669] +process[print_text-8]: started with pid [2021670] +process[rosbag_record-9]: started with pid [2021694] +started bagging! +[INFO] [1736042750.603480]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736042751.410116]: Stored episode 3. +[INFO] [1736042751.412748]: Saving video... +[INFO] [1736042751.936366]: Video saved! +Starting episode 3 +[INFO] [1736042751.987828]: resetting environment +[INFO] [1736042751.997557]: cleared current plan +[INFO] [1736042751.997794]: moving to home +[WARN] [1736042751.931553]: Plan exhausted +[WARN] [1736042751.971537]: Plan exhausted +[INFO] [1736042755.704580]: reached home +[INFO] [1736042755.708017]: reset action history +[INFO] [1736042758.447273]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042760.924906]: resetting environment +[INFO] [1736042760.938300]: cleared current plan +[INFO] [1736042760.939531]: moving to home +[INFO] [1736042761.942699]: reached home +[INFO] [1736042761.945292]: reset action history +[INFO] [1736042764.673441]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/episode_rosbags/episode_3_2025-01-04-21-06-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2021110.log +started roslaunch server http://158.130.50.37:44667/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2021817] +process[print_text-11]: started with pid [2021818] +process[rosbag_record-12]: started with pid [2021819] +started bagging! +[INFO] [1736042791.662123]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736042792.421315]: Stored episode 4. +[INFO] [1736042792.428602]: Saving video... +[INFO] [1736042792.714767]: Video saved! +Starting episode 4 +[INFO] [1736042792.765220]: resetting environment +[INFO] [1736042792.782582]: cleared current plan +[INFO] [1736042792.782817]: moving to home +[INFO] [1736042796.490098]: reached home +[INFO] [1736042796.494569]: reset action history +[INFO] [1736042799.308280]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042809.488603]: resetting environment +[INFO] [1736042809.499614]: cleared current plan +[INFO] [1736042809.502308]: moving to home +[INFO] [1736042810.507295]: reached home +[INFO] [1736042810.511338]: reset action history +[INFO] [1736042813.295433]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/episode_rosbags/episode_4_2025-01-04-21-06-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2021110.log +started roslaunch server http://158.130.50.37:41939/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2021986] +process[print_text-14]: started with pid [2021987] +process[rosbag_record-15]: started with pid [2021988] +started bagging! +For topic gripper_width: timestamp difference is 108117607 for nearest: 1736042815418225553 - target: 1736042815310107946 at idx 27 +For topic gripper_width: timestamp difference is 108957395 for nearest: 1736042815818254337 - target: 1736042815709296942 at idx 31 +For topic gripper_width: timestamp difference is 125877554 for nearest: 1736042824218228275 - target: 1736042824092350721 at idx 153 +For topic gripper_width: timestamp difference is 101107715 for nearest: 1736042838451557277 - target: 1736042838350449562 at idx 249 +[INFO] [1736042841.631438]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1736042842.369712]: Stored episode 5. +[INFO] [1736042842.380126]: Saving video... +[INFO] [1736042842.621419]: Video saved! +[INFO] [1736042842.684174]: resetting environment +[INFO] [1736042842.703902]: cleared current plan +[INFO] [1736042842.717394]: moving to home +[INFO] [1736042846.439238]: reached home +[INFO] [1736042846.453940]: reset action history +[INFO] [1736042849.327060]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-metadata.json b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a599da13fc1fc06835a3bc040032e204a24a4ac2 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:02:56.747830", + "startedAt": "2025-01-05T02:02:56.363221", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3941.48725, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.933, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.323, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 643.6809577941895 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6898b58a10093c2ec2e2df8496af514c7a12d2e8 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c8c6c007478e4fec430512ffde6b8aec6b9463b1fcdf04c4b085f1c37417e180", "size": 1155189, "path": "media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4"}, "global_step": 665, "_timestamp": 1736042858.034308, "_runtime": 281.65959191322327, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "cdfd18c4a1782d10be6a08ed84572c8662c780962ee66acd115847f54437d391", "size": 1156643, "path": "media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4"}, "eval/num_success": 5.0, "episode": 40.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "bf28e457796d51f7468174bc4b716fb4bdeb422c76f450a360ef4ba6308a13d4", "size": 1160912, "path": "media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "05a6da1b8b18548e98dc1b3ef6612140bfd2f89cde0d9200160c31bd04c78166", "size": 1153377, "path": "media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "4dd22cdf48d22cf5a74051cd94e780009189a307951996230c198f9d7b9c5c0a", "size": 1208016, "path": "media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://uwszbq96hojjnmr7jh81imzry1ifybwwkvvi69l5eg7ftggybrlb45efsapdw0attp1x5nln1fznzajfhrqxb83cxzo3uz6160v8x4liw4o845kcyfem08qbrg8w4egj/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://jgq851vfmieqbwy997m0pjjsj5tvar5o3exdnjz3a86qhtfdz5v7u8blffuau0xjuvxgoruivnl7a4rorrli6dnba1fi0akpkialtcry9fxo5253qls4x5u11dqbii8t:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 33, "total_num_episodes": 40, "total_success_rate": 0.825, "total_success_rate_lb_0.01": 0.6728713810098137, "total_success_rate_ub_0.01": 0.9291585086917867, "total_success_rate_lb_0.025": 0.6784497797132267, "total_success_rate_ub_0.025": 0.9146590530043026, "total_success_rate_lb_0.05": 0.6985354423125878, "total_success_rate_ub_0.05": 0.9084032847660557, "total_success_rate_lb_0.1": 0.7483667879799973, "total_success_rate_ub_0.1": 0.8952674660041156, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2353275f55aab57b5ba71dabb60150503bfd39ceb8d9ccf3c24f13550b1e45d3", "size": 507, "artifact_path": "wandb-client-artifact://yqkq6jl7yu71vhwtapyvjfevc64j11mleexnqcfl2tzbpc0ab6lws7jf3ru193co98eekiwynxltzle0qe3nqft8q6oiqo7yz582ej2fclqhyupsn8867grha1p3d2bi/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://r7vywht81hw90u2f5ap2ek1xrlv1kksd7kcibxjgv06xzijfeer46dutf0o2m6p8fbc1ydrjcrj2tsos86cc2cs1b45xe4bc54vbnu4ilwvxisk5hf9epx4m4v046kod:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 281}, "eval/total_success_rate": 0.824999988079071, "eval/total_success_rate_lb_0.01": 0.6728713512420654, "eval/total_success_rate_ub_0.01": 0.9291585087776184, "eval/total_success_rate_lb_0.025": 0.6784497499465942, "eval/total_success_rate_ub_0.025": 0.9146590232849121, "eval/total_success_rate_lb_0.05": 0.6985354423522949, "eval/total_success_rate_ub_0.05": 0.9084032773971558, "eval/total_success_rate_lb_0.1": 0.7483667731285095, "eval/total_success_rate_ub_0.1": 0.8952674865722656} \ No newline at end of file diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug-internal.log b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..87dee0c68e88ac4c79084688a3fdabcf4e6efabe --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug-internal.log @@ -0,0 +1,786 @@ +2025-01-04 21:02:56,374 INFO StreamThr :2021275 [internal.py:wandb_internal():85] W&B internal server running at pid: 2021275, started at: 2025-01-04 21:02:56.374648 +2025-01-04 21:02:56,376 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:02:56,386 INFO WriterThread:2021275 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/run-l2vooc6k.wandb +2025-01-04 21:02:56,387 DEBUG SenderThread:2021275 [sender.py:send():379] send: header +2025-01-04 21:02:56,395 DEBUG SenderThread:2021275 [sender.py:send():379] send: run +2025-01-04 21:02:56,642 INFO SenderThread:2021275 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files +2025-01-04 21:02:56,642 INFO SenderThread:2021275 [sender.py:_start_run_threads():1188] run started: l2vooc6k with start time 1736042576.374716 +2025-01-04 21:02:56,659 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:02:56,659 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:02:56,729 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:02:56,734 DEBUG HandlerThread:2021275 [system_info.py:__init__():26] System info init +2025-01-04 21:02:56,734 DEBUG HandlerThread:2021275 [system_info.py:__init__():41] System info init done +2025-01-04 21:02:56,734 INFO HandlerThread:2021275 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:02:56,734 INFO SystemMonitor:2021275 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:02:56,735 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:02:56,735 INFO HandlerThread:2021275 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:02:56,735 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:02:56,736 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:02:56,737 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:02:56,737 INFO SystemMonitor:2021275 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:02:56,747 DEBUG HandlerThread:2021275 [system_info.py:probe():152] Probing system +2025-01-04 21:02:56,749 DEBUG HandlerThread:2021275 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_info.py:probe():200] Probing system done +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:02:56.747830', 'startedAt': '2025-01-05T02:02:56.363221', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3941.48725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.933, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.323, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 643.6809577941895}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:02:56,754 INFO HandlerThread:2021275 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:02:56,754 INFO HandlerThread:2021275 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:02:56,754 DEBUG HandlerThread:2021275 [system_info.py:_save_code():44] Saving code +2025-01-04 21:02:56,756 DEBUG HandlerThread:2021275 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:02:56,756 DEBUG HandlerThread:2021275 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:02:56,818 DEBUG HandlerThread:2021275 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:02:56,818 INFO HandlerThread:2021275 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:02:56,822 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:02:56,822 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:02:56,822 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:02:56,822 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:02:57,074 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/p24k1bf2-wandb-metadata.json +2025-01-04 21:02:57,082 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:02:57,082 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:02:57,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:02:57,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:02:57,095 INFO wandb-upload_2:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/puvneh0w-diff.patch +2025-01-04 21:02:57,117 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/j71v2x6k-code/FISH/eval_robot.py +2025-01-04 21:02:57,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:57,125 INFO HandlerThread:2021275 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/tb" + save: true +} +control { + relay_id: "34850869c6cf4c14b3fc83ae75ff0b9f" +} +_info { + stream_id: "l2vooc6k" +} + +2025-01-04 21:02:57,125 DEBUG HandlerThread:2021275 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:02:57,236 DEBUG SenderThread:2021275 [sender.py:send():379] send: telemetry +2025-01-04 21:02:57,237 DEBUG SenderThread:2021275 [sender.py:send():379] send: tbrecord +2025-01-04 21:02:57,237 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:02:57,237 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 with policy live +2025-01-04 21:02:57,455 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/dde2c3ad-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/diff.patch +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-metadata.json +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH/eval_robot.py +2025-01-04 21:02:57,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:02:57,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:02:57,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code +2025-01-04 21:02:57,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH +2025-01-04 21:02:58,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:59,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:02:59,643 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:00,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:01,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:02,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:02,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:02,662 DEBUG SenderThread:2021275 [sender.py:send():379] send: config +2025-01-04 21:03:03,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:03,644 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:04,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:05,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:05,645 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:06,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:07,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:07,386 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:07,646 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:08,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:09,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:09,646 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:10,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:11,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:11,647 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:12,082 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:12,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:12,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:13,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:13,151 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:14,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:15,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:16,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:17,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:18,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:19,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:19,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:20,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:21,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:22,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:23,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:24,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:24,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:25,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:26,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:27,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:27,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:27,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:28,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:29,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:29,245 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:29,652 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml +2025-01-04 21:03:30,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:31,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:32,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:33,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:34,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:35,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:35,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:36,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:37,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:38,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:39,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:40,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:40,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:41,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:42,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:42,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:42,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:43,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:44,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:45,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:45,182 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:46,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:47,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:48,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:49,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:50,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:51,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:51,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:52,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:53,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:54,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:55,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:56,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:03:56,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:56,737 DEBUG SystemMonitor:2021275 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:03:56,738 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:03:57,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:03:57,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:03:57,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:57,660 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:03:58,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:59,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:03:59,660 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:00,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:01,091 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:01,661 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:01,831 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:02,094 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:03,094 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:03,661 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:04,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:05,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:06,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:07,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:07,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:08,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:09,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:10,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:11,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:12,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:12,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:12,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:12,142 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:13,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:14,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:15,096 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:16,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:17,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:18,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:18,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:19,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:20,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:21,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:22,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:23,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:24,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:24,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:25,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:26,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:26,738 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:04:27,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:27,083 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:27,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:28,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:28,490 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:04:28,490 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 with policy now +2025-01-04 21:04:28,490 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:04:28,491 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:04:28,491 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:04:28,492 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:04:28,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:04:28,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:04:28,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:04:28,669 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos +2025-01-04 21:04:28,669 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media +2025-01-04 21:04:28,757 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/mkpq3ptv-media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:04:29,012 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/ny3dotta-media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:04:29,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:29,502 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:29,668 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:30,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:31,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:32,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:33,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:33,669 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:34,109 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:34,919 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:35,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:36,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:37,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:37,672 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:38,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:39,110 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:39,673 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:39,673 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:04:39,914 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/o6fomjur-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:04:40,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:40,885 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:41,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:41,673 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:42,083 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:42,084 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:42,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:43,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:43,674 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:44,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:45,111 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:46,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:46,830 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:47,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:48,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:49,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:50,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:51,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:52,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:52,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:53,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:54,119 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:55,120 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:56,120 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:56,739 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:04:57,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:04:57,085 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:04:57,125 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:57,173 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:04:57,678 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:04:58,120 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:04:59,122 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:00,122 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:01,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:02,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:03,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:03,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:04,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:05,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:06,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:07,123 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:08,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:08,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:09,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:10,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:10,699 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:05:10,699 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 with policy now +2025-01-04 21:05:10,699 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:05:10,700 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:05:10,700 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:05:10,700 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:05:10,966 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/prn4p2zm-media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:05:11,124 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:11,682 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:05:11,683 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:11,683 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:05:11,683 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:05:11,974 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/zajpxdfm-media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:05:12,086 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:12,086 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:12,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:13,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:13,190 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:14,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:15,134 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:15,684 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:16,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:17,128 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:18,127 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:18,245 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:19,127 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:19,685 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:20,128 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:20,586 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:05:20,586 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:05:20,587 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:05:20,685 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:05:20,686 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:05:21,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:21,686 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:22,128 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:23,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:23,609 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:23,687 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:24,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:25,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:25,687 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:26,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:26,739 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:05:27,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:27,084 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:27,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:28,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:29,129 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:29,231 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:30,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:31,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:32,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:33,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:34,130 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:35,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:35,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:36,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:37,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:38,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:39,131 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:40,088 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:40,132 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:41,132 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:42,084 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:42,084 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:42,132 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:43,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:44,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:45,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:45,161 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:46,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:47,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:48,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:49,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:50,133 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:50,604 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:51,135 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:51,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:51,978 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:05:51,978 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 with policy now +2025-01-04 21:05:51,981 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:05:51,982 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:05:51,982 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:05:51,982 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:05:52,134 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:52,240 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/4v6ig217-media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:05:52,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:05:52,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:05:52,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:05:52,982 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/vmr8xpxz-media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:05:53,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:53,696 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:54,153 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:54,697 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:55,140 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:55,705 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:05:56,141 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:56,740 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:05:57,090 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:05:57,090 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:05:57,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:57,698 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:05:58,146 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:59,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:05:59,698 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:00,146 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:00,900 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:01,142 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:01,400 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:06:01,401 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:06:01,401 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:06:01,699 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:06:01,699 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:01,699 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:06:01,948 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/shnt39x8-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:06:02,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:03,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:03,700 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:04,150 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:05,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:05,701 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:05,941 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:06,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:07,144 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:08,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:09,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:10,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:11,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:11,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:12,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:12,085 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:12,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:13,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:14,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:15,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:16,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:16,173 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:17,145 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:18,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:19,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:20,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:21,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:22,089 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:22,147 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:23,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:24,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:25,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:26,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:26,740 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:06:27,085 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:27,086 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:27,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:27,198 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:28,148 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:29,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:30,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:31,149 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:32,172 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:32,424 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:32,752 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:06:32,753 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 with policy now +2025-01-04 21:06:32,762 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:06:32,763 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:06:32,763 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:06:32,763 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:06:33,016 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/efsavywt-media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:06:33,150 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:06:33,711 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:06:34,033 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/705cstlg-media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:06:34,153 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:35,158 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:36,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:37,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:37,712 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:38,097 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:38,170 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:39,156 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:40,161 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:41,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:41,714 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:42,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:42,095 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:42,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:43,160 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:43,196 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:44,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:45,167 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:46,162 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:47,162 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:48,164 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:49,098 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:49,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:49,716 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:06:50,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:50,458 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:06:50,458 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:06:50,459 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:06:50,717 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:06:51,163 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:51,717 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:52,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:53,165 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:54,174 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:54,341 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:06:55,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:55,718 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:56,167 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:56,741 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:06:57,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:06:57,087 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:06:57,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:57,719 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:06:58,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:06:59,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:00,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:00,228 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:01,166 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:02,169 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:03,169 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:04,169 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:05,170 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:05,265 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:05,722 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:06,174 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:07,174 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:08,177 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:09,177 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:10,178 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:11,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:11,178 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:12,087 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:07:12,087 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:07:12,178 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:13,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:14,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:15,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:16,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:16,189 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:17,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:18,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:19,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:19,727 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:20,180 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:21,179 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:21,501 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:22,191 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:22,682 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:07:22,682 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 with policy now +2025-01-04 21:07:22,685 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:07:22,685 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:22,685 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:22,685 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:22,728 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:22,728 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:22,729 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval +2025-01-04 21:07:22,946 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/ojvaljad-media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:23,199 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:23,215 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/joaa1t3v-media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:23,728 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:24,202 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:25,196 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:26,200 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:26,741 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:07:26,741 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:27,095 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:07:27,096 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:07:27,203 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:27,730 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:28,200 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:29,216 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:30,204 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:31,199 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:31,731 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:32,099 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:32,203 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:33,205 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:34,215 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:35,208 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:36,218 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:37,218 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:37,481 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:07:37,481 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:07:37,700 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:07:37,733 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:37,733 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table +2025-01-04 21:07:37,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval +2025-01-04 21:07:37,922 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/q79izaar-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:37,992 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/qcg0dzhm-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:38,022 INFO wandb-upload_2:2021275 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprehdr76g +2025-01-04 21:07:38,116 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:07:38,127 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:07:38,216 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:38,394 INFO SenderThread:2021275 [sender.py:send_request_log_artifact():1525] logged artifact run-l2vooc6k-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzNTY5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzAwNw==', 'latestArtifact': None}} +2025-01-04 21:07:38,395 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:38,395 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,395 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:38,395 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:07:38,396 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,396 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,396 DEBUG SenderThread:2021275 [sender.py:send():379] send: telemetry +2025-01-04 21:07:38,397 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,397 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,397 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,397 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,398 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,398 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,398 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,398 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,399 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,400 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,401 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,401 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,401 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,401 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,402 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,402 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,402 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,402 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,402 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,403 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,403 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:38,403 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:38,403 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:07:38,634 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/b1yjdlo0-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:38,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval +2025-01-04 21:07:38,794 INFO wandb-upload_2:2021275 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfjmsyvm8 +2025-01-04 21:07:38,925 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/13v7wzvc-media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:39,162 INFO SenderThread:2021275 [sender.py:send_request_log_artifact():1525] logged artifact run-l2vooc6k-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEzNTcxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzAxMg==', 'latestArtifact': None}} +2025-01-04 21:07:39,162 DEBUG SenderThread:2021275 [sender.py:send():379] send: files +2025-01-04 21:07:39,162 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json with policy now +2025-01-04 21:07:39,162 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:39,163 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:39,163 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:39,163 DEBUG SenderThread:2021275 [sender.py:send():379] send: exit +2025-01-04 21:07:39,163 INFO SenderThread:2021275 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:07:39,164 INFO SenderThread:2021275 [sender.py:send_exit():588] handling runtime: 281 +2025-01-04 21:07:39,167 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:send_exit():594] send defer +2025-01-04 21:07:39,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:07:39,168 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:07:39,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:07:39,168 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:07:39,168 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:07:39,168 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:07:39,168 INFO HandlerThread:2021275 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:07:39,169 DEBUG SystemMonitor:2021275 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:07:39,169 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:07:39,169 DEBUG SystemMonitor:2021275 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:07:39,169 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:07:39,222 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:07:39,222 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:07:39,222 INFO HandlerThread:2021275 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:07:39,223 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,223 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:07:39,223 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:07:39,223 DEBUG SenderThread:2021275 [sender.py:send():379] send: stats +2025-01-04 21:07:39,223 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,223 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:07:39,223 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:39,223 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:07:39,224 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:07:39,224 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:39,224 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:07:39,398 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/x1rf1xvc-media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:39,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:39,734 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:41,735 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:45,521 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:45,521 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:45,521 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,522 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:07:45,522 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send():379] send: history +2025-01-04 21:07:45,522 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:07:45,523 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:45,523 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:07:45,523 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:45,524 DEBUG SenderThread:2021275 [sender.py:send():379] send: summary +2025-01-04 21:07:45,527 INFO SenderThread:2021275 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:07:45,527 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,527 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:07:45,527 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:07:45,527 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:45,528 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:07:45,528 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,528 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:07:45,531 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:07:45,666 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:07:45,667 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:45,667 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:07:45,667 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:45,667 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:07:45,737 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:45,737 INFO Thread-12 :2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml +2025-01-04 21:07:46,515 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:48,045 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:07:48,045 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:48,045 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:48,046 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:07:48,046 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:48,046 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:07:48,046 INFO SenderThread:2021275 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:07:48,048 INFO SenderThread:2021275 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:07:48,050 INFO SenderThread:2021275 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:07:48,064 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:07:48,065 DEBUG SenderThread:2021275 [sender.py:send():379] send: artifact +2025-01-04 21:07:48,067 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:48,068 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:07:48,237 INFO SenderThread:2021275 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:07:48,237 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:48,237 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:07:48,237 INFO SenderThread:2021275 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:07:48,515 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml config.yaml +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json wandb-summary.json +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt requirements.txt +2025-01-04 21:07:48,738 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:07:48,739 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log output.log +2025-01-04 21:07:48,739 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/diff.patch diff.patch +2025-01-04 21:07:48,743 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:07:48,743 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 media/videos/eval/3_eval_3_05a6da1b8b18548e98dc.mp4 +2025-01-04 21:07:48,743 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 media/videos/eval/4_eval_4_4dd22cdf48d22cf5a740.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 media/videos/eval/2_eval_2_bf28e457796d51f74681.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 media/videos/eval/0_eval_0_c8c6c007478e4fec4305.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 media/videos/eval/1_eval_1_cdfd18c4a1782d10be6a.mp4 +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json media/table/eval/total_success_rate_ci_6_2353275f55aab57b5ba7.table.json +2025-01-04 21:07:48,744 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:07:48,744 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:48,744 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:48,744 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:07:48,749 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:48,749 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:07:48,749 INFO SenderThread:2021275 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:07:48,928 INFO wandb-upload_2:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/wandb-summary.json +2025-01-04 21:07:48,949 INFO wandb-upload_1:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/config.yaml +2025-01-04 21:07:48,952 INFO wandb-upload_3:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/requirements.txt +2025-01-04 21:07:48,984 INFO wandb-upload_4:2021275 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/files/output.log +2025-01-04 21:07:49,018 INFO wandb-upload_0:2021275 [upload_job.py:push():130] Uploaded file /tmp/tmpfgzsk0y0wandb/9hue959f-events.out.tfevents.1736042577.leonmkim-ROG-Strix-G15CS-G15CS.2021110.0 +2025-01-04 21:07:49,219 INFO Thread-11 (_thread_body):2021275 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:07:49,219 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,219 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:07:49,219 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,219 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:07:49,220 INFO SenderThread:2021275 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:07:49,221 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:07:49,221 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,221 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:07:49,221 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,221 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:07:49,221 INFO SenderThread:2021275 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:07:49,446 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,446 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:07:49,446 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:07:49,446 INFO SenderThread:2021275 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:07:49,446 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:07:49,446 INFO HandlerThread:2021275 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:07:49,447 DEBUG SenderThread:2021275 [sender.py:send():379] send: final +2025-01-04 21:07:49,447 DEBUG SenderThread:2021275 [sender.py:send():379] send: footer +2025-01-04 21:07:49,447 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: defer +2025-01-04 21:07:49,447 INFO SenderThread:2021275 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:07:49,448 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:07:49,449 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:07:49,449 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:49,450 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:07:49,451 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:07:49,498 DEBUG SenderThread:2021275 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:07:49,498 INFO MainThread:2021275 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:07:49,498 INFO MainThread:2021275 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:07:49,498 INFO MainThread:2021275 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:07:49,499 DEBUG HandlerThread:2021275 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:07:49,499 INFO HandlerThread:2021275 [handler.py:finish():882] shutting down handler +2025-01-04 21:07:50,449 INFO WriterThread:2021275 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/run-l2vooc6k.wandb +2025-01-04 21:07:50,498 INFO SenderThread:2021275 [sender.py:finish():1615] shutting down sender +2025-01-04 21:07:50,498 INFO SenderThread:2021275 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:07:50,498 INFO SenderThread:2021275 [file_pusher.py:join():175] waiting for file pusher diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug.log b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7cb5e7fcd4274fc0f9d27cc287dedc1e3be711e2 --- /dev/null +++ b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Configure stats pid to 2021110 +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/settings +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug.log +2025-01-04 21:02:56,368 INFO MainThread:2021110 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/logs/debug-internal.log +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k'} +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():619] starting backend +2025-01-04 21:02:56,369 INFO MainThread:2021110 [wandb_init.py:init():623] setting up manager +2025-01-04 21:02:56,373 INFO MainThread:2021110 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:02:56,374 INFO MainThread:2021110 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:02:56,386 INFO MainThread:2021110 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:02:56,393 INFO MainThread:2021110 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:02:56,659 INFO MainThread:2021110 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:02:56,724 INFO MainThread:2021110 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:02:56,725 INFO MainThread:2021110 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:02:57,082 INFO MainThread:2021110 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:02:57,083 INFO MainThread:2021110 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:02:57,083 INFO MainThread:2021110 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:02:57,083 INFO MainThread:2021110 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:02:57,085 INFO MainThread:2021110 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:02:57,085 INFO MainThread:2021110 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/l2vooc6k/tb, True +2025-01-04 21:03:02,662 INFO MainThread:2021110 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 21:07:50,499 WARNING MsgRouterThr:2021110 [router.py:message_loop():77] message_loop has been closed diff --git a/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/run-l2vooc6k.wandb b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/run-l2vooc6k.wandb new file mode 100644 index 0000000000000000000000000000000000000000..25bc31be1ddd69cf871952272dd081d04b81d059 Binary files /dev/null and b/l2vooc6k/wandb/run-20250104_210256-l2vooc6k/run-l2vooc6k.wandb differ diff --git a/ot5wit9y/.hydra/config.yaml b/ot5wit9y/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/ot5wit9y/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ot5wit9y/.hydra/hydra.yaml b/ot5wit9y/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89969ad835d940a1ae76ea88fcfe2b43f9426256 --- /dev/null +++ b/ot5wit9y/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ot5wit9y/.hydra/overrides.yaml b/ot5wit9y/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ot5wit9y/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ot5wit9y/episode_rosbags/aligned_depth_to_color_K.npy b/ot5wit9y/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ot5wit9y/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ot5wit9y/episode_rosbags/cam_tf_world.npy b/ot5wit9y/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/ot5wit9y/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/ot5wit9y/episode_rosbags/color_K.npy b/ot5wit9y/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ot5wit9y/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ot5wit9y/episode_rosbags/depth_K.npy b/ot5wit9y/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/ot5wit9y/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/ot5wit9y/episode_rosbags/episode_0_2025-01-04-21-09-58.bag b/ot5wit9y/episode_rosbags/episode_0_2025-01-04-21-09-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..acf893cd3a685f48984b815e8acf1643663cf1c0 --- /dev/null +++ b/ot5wit9y/episode_rosbags/episode_0_2025-01-04-21-09-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef2fedd913e830a1f795b0fcb089010160b919938cb3d35ea636377e43a3a245 +size 1408522223 diff --git a/ot5wit9y/episode_rosbags/episode_1_2025-01-04-21-10-39.bag b/ot5wit9y/episode_rosbags/episode_1_2025-01-04-21-10-39.bag new file mode 100644 index 0000000000000000000000000000000000000000..bf4ff0ce6500ccd98a3e7d460e8baeaceae705aa --- /dev/null +++ b/ot5wit9y/episode_rosbags/episode_1_2025-01-04-21-10-39.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80cfd4005eb0dd6968b0fcc0b46c58b4db5f50a6d91e3053f912e00fc16636f9 +size 1414050949 diff --git a/ot5wit9y/episode_rosbags/episode_2_2025-01-04-21-11-30.bag b/ot5wit9y/episode_rosbags/episode_2_2025-01-04-21-11-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..3a290121fe0e478b8475b27776578df34457c086 --- /dev/null +++ b/ot5wit9y/episode_rosbags/episode_2_2025-01-04-21-11-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882d8c6ed415c13f00c206ab33f0967e077dc45e9d874ac6f136239d26898dc7 +size 1411527253 diff --git a/ot5wit9y/episode_rosbags/episode_3_2025-01-04-21-12-11.bag b/ot5wit9y/episode_rosbags/episode_3_2025-01-04-21-12-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..87a31e256b818876d3582e3ab56b602b6f533d56 --- /dev/null +++ b/ot5wit9y/episode_rosbags/episode_3_2025-01-04-21-12-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1b1425fd5d653f1b7c388453d720738f9352a050a9d84f64a3ddd1e146ebb12 +size 1413922675 diff --git a/ot5wit9y/episode_rosbags/episode_4_2025-01-04-21-12-56.bag b/ot5wit9y/episode_rosbags/episode_4_2025-01-04-21-12-56.bag new file mode 100644 index 0000000000000000000000000000000000000000..75a562f37c374c469f5450d4862adb3bc06ec21d --- /dev/null +++ b/ot5wit9y/episode_rosbags/episode_4_2025-01-04-21-12-56.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30bc941b3159256a85ebcfa4bdf1a298dab8993dbc88484b904492154056fabd +size 1409634772 diff --git a/ot5wit9y/eval_robot.log b/ot5wit9y/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..a908b0550709cf4f029e58307868791eab89c5a0 --- /dev/null +++ b/ot5wit9y/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 21:09:25,106][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:09:25,109][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/ot5wit9y/eval_video/0_eval.mp4 b/ot5wit9y/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2325e0e767e5e5618dea984d38b53b41ae05041a --- /dev/null +++ b/ot5wit9y/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:258e75c4654817391a5f92df0dfa08592a335185821d354a0825c0235cede97a +size 1153786 diff --git a/ot5wit9y/eval_video/1_eval.mp4 b/ot5wit9y/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d2e2a2f343d95d61d4b726df09e00463fc0f40eb --- /dev/null +++ b/ot5wit9y/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3521872760c4883ab429067c704af7d3051524dc89faa08875d6edd55fe3cd1 +size 1187188 diff --git a/ot5wit9y/eval_video/2_eval.mp4 b/ot5wit9y/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..66fd93a9f640ca1c15b46c260eec2dcf3fdf4161 --- /dev/null +++ b/ot5wit9y/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:941ebe96bf32b79e70bb907e73c85a0c662ae695e8c381c1470942ee69da5d32 +size 1158306 diff --git a/ot5wit9y/eval_video/3_eval.mp4 b/ot5wit9y/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9bad5794ea91e9238762bb02e5f1cf0263538e2 --- /dev/null +++ b/ot5wit9y/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a15a18ebb9ab075df26a7f49d825502554eebb85f6ab2d1a5919a43a6ad919 +size 1170367 diff --git a/ot5wit9y/eval_video/4_eval.mp4 b/ot5wit9y/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7221162175b18ffe53fdb7ee32aee12d5e04fd7 --- /dev/null +++ b/ot5wit9y/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f55a68bbcaedd47cf64e6b3feec8facbf88d1b4d6a0e5cad6e5a213a85cd27dd +size 1166605 diff --git a/ot5wit9y/tb/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 b/ot5wit9y/tb/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 new file mode 100644 index 0000000000000000000000000000000000000000..e222b28bec8c17b34e31a34aa6dc1a6feca66a9e --- /dev/null +++ b/ot5wit9y/tb/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fa63b97a13be7bb1c4b54a838a3985d980c3476ececf9b19fedcf143e935c4f +size 6539 diff --git a/ot5wit9y/wandb/debug-internal.log b/ot5wit9y/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2c16f2d87e44328491af332c774d52c0475f66c6 --- /dev/null +++ b/ot5wit9y/wandb/debug-internal.log @@ -0,0 +1,770 @@ +2025-01-04 21:09:30,841 INFO StreamThr :2022441 [internal.py:wandb_internal():85] W&B internal server running at pid: 2022441, started at: 2025-01-04 21:09:30.840683 +2025-01-04 21:09:30,841 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:09:30,853 INFO WriterThread:2022441 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/run-ot5wit9y.wandb +2025-01-04 21:09:30,854 DEBUG SenderThread:2022441 [sender.py:send():379] send: header +2025-01-04 21:09:30,865 DEBUG SenderThread:2022441 [sender.py:send():379] send: run +2025-01-04 21:09:31,115 INFO SenderThread:2022441 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files +2025-01-04 21:09:31,115 INFO SenderThread:2022441 [sender.py:_start_run_threads():1188] run started: ot5wit9y with start time 1736042970.841331 +2025-01-04 21:09:31,132 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:09:31,132 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:09:31,207 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:09:31,212 DEBUG HandlerThread:2022441 [system_info.py:__init__():26] System info init +2025-01-04 21:09:31,212 DEBUG HandlerThread:2022441 [system_info.py:__init__():41] System info init done +2025-01-04 21:09:31,212 INFO HandlerThread:2022441 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:09:31,212 INFO SystemMonitor:2022441 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:09:31,212 INFO HandlerThread:2022441 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:09:31,213 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:09:31,213 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:09:31,213 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:09:31,214 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:09:31,215 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:09:31,224 DEBUG HandlerThread:2022441 [system_info.py:probe():152] Probing system +2025-01-04 21:09:31,226 DEBUG HandlerThread:2022441 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:09:31,232 DEBUG HandlerThread:2022441 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:09:31,232 DEBUG HandlerThread:2022441 [system_info.py:probe():200] Probing system done +2025-01-04 21:09:31,232 DEBUG HandlerThread:2022441 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:09:31.225064', 'startedAt': '2025-01-05T02:09:30.827805', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.0071249999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.971, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 650.2632446289062}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:09:31,233 INFO HandlerThread:2022441 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:09:31,233 INFO HandlerThread:2022441 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:09:31,233 DEBUG HandlerThread:2022441 [system_info.py:_save_code():44] Saving code +2025-01-04 21:09:31,234 DEBUG HandlerThread:2022441 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:09:31,234 DEBUG HandlerThread:2022441 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:09:31,297 DEBUG HandlerThread:2022441 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:09:31,297 INFO HandlerThread:2022441 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:09:31,300 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:09:31,300 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:09:31,301 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:09:31,301 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:09:31,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:09:31,535 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:09:31,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:31,537 DEBUG SenderThread:2022441 [sender.py:send():379] send: telemetry +2025-01-04 21:09:31,544 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/s90pkjn7-wandb-metadata.json +2025-01-04 21:09:31,571 INFO wandb-upload_2:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/fnuvzh0w-diff.patch +2025-01-04 21:09:31,577 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:09:31,577 INFO HandlerThread:2022441 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/tb" + save: true +} +control { + relay_id: "0e53d899c52a40d3991c399e0b797cce" +} +_info { + stream_id: "ot5wit9y" +} + +2025-01-04 21:09:31,577 DEBUG HandlerThread:2022441 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:09:31,578 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:09:31,580 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/k7l7ltps-code/FISH/eval_robot.py +2025-01-04 21:09:31,738 DEBUG SenderThread:2022441 [sender.py:send():379] send: tbrecord +2025-01-04 21:09:31,738 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:09:31,739 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 with policy live +2025-01-04 21:09:31,925 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/vieyoom2-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH/eval_robot.py +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-metadata.json +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/diff.patch +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code +2025-01-04 21:09:32,118 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH +2025-01-04 21:09:32,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:33,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:34,116 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:34,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:35,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:36,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:36,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:37,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:38,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:39,409 DEBUG SenderThread:2022441 [sender.py:send():379] send: config +2025-01-04 21:09:39,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:40,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:41,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:41,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:42,118 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:42,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:43,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:44,119 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:44,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:45,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:46,120 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:46,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:09:46,535 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:09:46,577 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:46,597 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:47,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:48,120 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:48,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:49,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:50,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:51,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:52,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:52,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:53,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:54,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:55,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:56,122 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:56,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:57,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:57,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:58,123 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:58,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:59,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:00,123 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:00,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:01,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:01,540 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:01,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:02,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:02,660 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:03,124 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml +2025-01-04 21:10:03,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:04,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:05,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:06,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:07,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:08,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:08,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:09,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:10,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:11,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:12,127 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:12,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:13,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:13,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:14,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:15,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:16,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:16,538 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:16,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:17,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:18,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:18,599 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:19,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:20,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:21,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:22,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:23,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:24,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:24,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:25,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:26,117 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:10:26,117 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 with policy now +2025-01-04 21:10:26,117 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:10:26,118 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:10:26,118 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:10:26,119 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:10:26,130 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:26,130 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media +2025-01-04 21:10:26,355 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/6avqu09j-media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:10:26,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:26,634 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/ul4148xg-media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:10:27,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:28,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:28,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:29,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:29,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:30,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:31,215 DEBUG SystemMonitor:2022441 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:10:31,216 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:10:31,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:31,538 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:31,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:32,132 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:32,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:33,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:34,133 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:34,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:35,482 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:35,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:36,134 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:10:36,372 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/wvlxpzyv-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:10:36,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:37,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:38,134 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:38,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:39,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:40,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:41,075 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:41,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:42,136 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:42,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:43,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:44,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:45,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:46,137 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:46,218 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:46,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:46,539 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:46,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:47,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:48,549 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:49,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:50,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:51,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:51,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:52,139 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:52,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:53,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:54,140 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:54,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:55,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:56,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:56,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:57,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:58,553 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:59,553 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:00,142 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:00,553 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:01,219 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:11:01,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:01,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:01,540 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:01,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:02,554 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:03,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:04,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:05,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:06,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:06,773 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:07,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:08,146 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:08,240 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:11:08,240 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 with policy now +2025-01-04 21:11:08,242 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:11:08,243 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:11:08,243 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:11:08,243 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:11:08,461 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/0ykbl0r1-media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:11:08,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:09,146 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:11:09,147 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:11:09,147 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:11:09,411 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/w54k1dwn-media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:11:09,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:10,147 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:10,556 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:11,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:11,863 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:12,556 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:13,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:14,148 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:14,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:15,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:16,149 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:16,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:16,539 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:16,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:17,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:17,698 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:18,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:19,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:20,150 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:20,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:21,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:22,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:23,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:23,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:24,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:25,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:26,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:27,152 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:11:27,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:27,996 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:11:27,996 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:11:27,996 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:11:28,153 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:11:28,153 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:28,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:28,997 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:29,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:30,154 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:30,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:31,217 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:11:31,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:31,539 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:31,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:32,154 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:32,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:33,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:34,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:34,625 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:35,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:36,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:37,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:38,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:39,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:40,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:40,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:41,562 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:42,562 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:43,571 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:44,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:45,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:46,159 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:46,461 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:46,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:46,542 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:46,589 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:47,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:48,160 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:48,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:49,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:50,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:51,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:51,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:52,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:53,573 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:54,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:55,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:56,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:57,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:57,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:58,577 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:58,925 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:11:58,925 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 with policy now +2025-01-04 21:11:58,926 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:11:58,928 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:11:58,928 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:11:58,928 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:11:59,156 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/iato1nkl-media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:11:59,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:11:59,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:11:59,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:11:59,417 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/zipbhv4v-media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:11:59,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:00,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:00,580 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:01,218 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:12:01,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:01,545 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:01,585 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:02,564 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:02,605 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:03,588 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:04,165 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:04,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:05,589 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:06,165 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:06,594 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:07,599 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:07,727 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:07,822 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:12:07,822 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:12:07,832 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:12:08,166 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:12:08,166 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:12:08,398 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/bn746x97-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:12:08,597 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:09,594 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:10,167 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:10,595 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:11,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:12,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:12,882 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:13,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:14,168 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:14,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:15,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:16,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:16,543 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:16,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:17,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:18,170 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:18,218 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:18,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:19,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:20,171 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:20,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:21,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:22,171 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:22,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:23,549 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:23,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:24,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:25,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:26,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:27,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:28,550 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:28,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:29,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:30,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:31,219 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:12:31,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:31,543 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:31,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:32,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:33,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:33,679 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:34,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:35,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:36,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:37,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:38,179 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:38,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:38,684 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:39,595 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:40,118 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:12:40,118 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 with policy now +2025-01-04 21:12:40,124 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:12:40,125 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:12:40,125 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:12:40,125 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:12:40,180 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:12:40,180 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:40,181 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:12:40,181 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:12:40,360 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/shzihwmr-media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:12:40,594 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:40,612 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/tc1mvs7u-media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:12:41,603 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:42,180 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:42,602 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:43,606 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:43,754 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:44,607 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:45,608 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:46,182 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:46,548 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:46,548 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:46,603 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:47,605 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:48,182 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:48,607 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:49,607 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:49,675 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:50,606 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:51,609 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:52,616 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:53,184 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:12:53,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:53,881 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:12:53,881 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:12:53,882 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:12:54,184 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:12:54,185 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:54,610 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:55,085 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:55,619 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:56,185 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:56,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:57,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:58,187 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:58,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:59,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:00,188 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:00,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:00,761 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:01,219 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:13:01,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:01,544 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:01,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:02,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:03,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:04,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:05,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:06,190 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:06,361 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:06,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:07,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:08,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:09,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:10,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:11,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:11,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:12,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:13,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:14,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:15,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:16,193 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:16,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:16,545 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:16,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:16,667 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:17,615 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:18,615 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:19,616 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:20,618 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:21,617 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:22,195 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:22,556 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:22,617 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:23,618 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:24,196 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:24,626 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:25,391 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:13:25,391 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 with policy now +2025-01-04 21:13:25,401 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:13:25,401 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:25,401 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:25,402 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:25,614 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/2aj43i21-media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:25,620 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:13:26,470 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/zzouipef-media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:26,633 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:27,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:27,632 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:28,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:29,628 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:30,198 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:30,632 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:31,220 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:13:31,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:31,559 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:31,639 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:32,639 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:32,917 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:33,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:34,200 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:34,639 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:35,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:36,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:37,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:38,110 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:38,201 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:38,443 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/f6zo02ej-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:38,635 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:38,936 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:38,937 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:38,937 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:39,065 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:13:39,065 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:13:39,201 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table +2025-01-04 21:13:39,274 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:13:39,385 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/cpmh7585-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:39,635 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:39,653 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:13:39,692 INFO wandb-upload_1:2022441 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmph02m9aq9 +2025-01-04 21:13:39,697 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval +2025-01-04 21:13:40,286 INFO SenderThread:2022441 [sender.py:send_request_log_artifact():1525] logged artifact run-ot5wit9y-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0MDYxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzY5NA==', 'latestArtifact': None}} +2025-01-04 21:13:40,286 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:13:40,286 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:13:40,286 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:40,286 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,287 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,287 DEBUG SenderThread:2022441 [sender.py:send():379] send: telemetry +2025-01-04 21:13:40,287 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,291 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,291 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,291 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,292 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,292 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,292 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,292 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,293 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,293 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,293 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,293 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,294 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,294 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,294 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,294 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,295 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,295 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,295 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,295 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,296 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,296 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,296 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:13:40,466 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/gyjh6vov-media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:40,563 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/s873ui01-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:40,635 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:40,656 INFO wandb-upload_2:2022441 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl783p45p +2025-01-04 21:13:41,085 INFO SenderThread:2022441 [sender.py:send_request_log_artifact():1525] logged artifact run-ot5wit9y-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0MDYzMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzY5Ng==', 'latestArtifact': None}} +2025-01-04 21:13:41,085 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:13:41,085 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json with policy now +2025-01-04 21:13:41,085 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:41,085 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:41,086 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:41,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:41,354 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/1qjb1onb-media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:41,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:42,203 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:42,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:43,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:43,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:44,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:45,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:46,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:46,545 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:46,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:47,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:48,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:48,642 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:49,206 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml +2025-01-04 21:13:49,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:50,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:51,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:51,741 DEBUG SenderThread:2022441 [sender.py:send():379] send: exit +2025-01-04 21:13:51,741 INFO SenderThread:2022441 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:13:51,742 INFO SenderThread:2022441 [sender.py:send_exit():588] handling runtime: 260 +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:send_exit():594] send defer +2025-01-04 21:13:51,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,743 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:13:51,743 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:13:51,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,743 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:13:51,744 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,744 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:13:51,744 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:13:51,744 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:13:51,744 DEBUG SystemMonitor:2022441 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:13:51,744 DEBUG SystemMonitor:2022441 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:13:51,798 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:13:51,798 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:13:51,798 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:13:51,798 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:13:51,799 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:13:51,799 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,799 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:13:51,799 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:13:51,799 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,799 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:13:52,207 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:54,208 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:57,943 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:57,943 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,943 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:57,943 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:13:57,944 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:13:57,944 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:13:57,946 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:57,946 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:57,946 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:13:57,947 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:57,947 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:57,948 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:57,948 DEBUG SenderThread:2022441 [sender.py:send():379] send: summary +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:57,952 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:13:57,952 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:57,952 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:13:57,952 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:13:57,952 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:57,953 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:57,953 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:13:57,953 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,953 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:13:58,105 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:13:58,105 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:58,105 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:13:58,105 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:58,105 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:13:58,105 INFO SenderThread:2022441 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:13:58,108 INFO SenderThread:2022441 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:13:58,110 INFO SenderThread:2022441 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:13:58,119 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:13:58,119 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:58,119 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:13:58,120 DEBUG SenderThread:2022441 [sender.py:send():379] send: artifact +2025-01-04 21:13:58,210 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:58,210 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:58,282 INFO SenderThread:2022441 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:13:58,282 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:58,282 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:13:58,282 INFO SenderThread:2022441 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:13:58,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,210 INFO SenderThread:2022441 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml config.yaml +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json wandb-summary.json +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt requirements.txt +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log output.log +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/diff.patch diff.patch +2025-01-04 21:13:59,214 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:59,216 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:13:59,219 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:13:59,220 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,220 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,220 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:13:59,220 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:13:59,393 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:59,401 INFO wandb-upload_2:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml +2025-01-04 21:13:59,429 INFO wandb-upload_4:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:59,443 INFO wandb-upload_3:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt +2025-01-04 21:13:59,467 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/y0qhm7vn-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:59,667 INFO Thread-11 (_thread_body):2022441 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:13:59,668 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,668 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:13:59,668 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,668 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:13:59,668 INFO SenderThread:2022441 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:13:59,669 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:13:59,669 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,669 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:13:59,670 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,670 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:13:59,670 INFO SenderThread:2022441 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:13:59,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,888 INFO SenderThread:2022441 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:13:59,888 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:13:59,888 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,889 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,889 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,889 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:13:59,889 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:13:59,889 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,889 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send():379] send: final +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send():379] send: footer +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,890 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:13:59,890 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,890 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,890 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:13:59,891 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:13:59,892 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,892 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:13:59,893 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:13:59,893 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:59,952 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,953 INFO MainThread:2022441 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:13:59,953 INFO MainThread:2022441 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:13:59,953 INFO MainThread:2022441 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:13:59,953 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:13:59,954 INFO HandlerThread:2022441 [handler.py:finish():882] shutting down handler +2025-01-04 21:14:00,893 INFO WriterThread:2022441 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/run-ot5wit9y.wandb +2025-01-04 21:14:00,953 INFO SenderThread:2022441 [sender.py:finish():1615] shutting down sender +2025-01-04 21:14:00,953 INFO SenderThread:2022441 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:14:00,953 INFO SenderThread:2022441 [file_pusher.py:join():175] waiting for file pusher diff --git a/ot5wit9y/wandb/debug.log b/ot5wit9y/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..af6bce55aa0ee2a611768b6851cc5f5b49ac3901 --- /dev/null +++ b/ot5wit9y/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:09:30,833 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Configure stats pid to 2022276 +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/settings +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug.log +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug-internal.log +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y'} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():619] starting backend +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():623] setting up manager +2025-01-04 21:09:30,839 INFO MainThread:2022276 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:09:30,840 INFO MainThread:2022276 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:09:30,853 INFO MainThread:2022276 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:09:30,862 INFO MainThread:2022276 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:09:31,131 INFO MainThread:2022276 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:09:31,201 INFO MainThread:2022276 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:09:31,201 INFO MainThread:2022276 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:09:31,535 INFO MainThread:2022276 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:09:31,535 INFO MainThread:2022276 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:09:31,536 INFO MainThread:2022276 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:09:31,536 INFO MainThread:2022276 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:09:31,537 INFO MainThread:2022276 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:09:31,538 INFO MainThread:2022276 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/tb, True +2025-01-04 21:09:39,408 INFO MainThread:2022276 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:14:00,954 WARNING MsgRouterThr:2022276 [router.py:message_loop():77] message_loop has been closed diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH/eval_robot.py b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b7084118e8ebb513167f114b593eadcc6a1ab02 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736042970 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/diff.patch b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c3325262d7f55687199a01005fa2a0cff23bbaee --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[38, 45, 0.8444444444444444, 0.7056096195763404, 0.9374039321263025, 0.7111262380654559, 0.9245381168172053, 0.729622602416844, 0.9189273825940081, 0.7748437188016042, 0.9072100165416974]]} \ No newline at end of file diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2325e0e767e5e5618dea984d38b53b41ae05041a --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:258e75c4654817391a5f92df0dfa08592a335185821d354a0825c0235cede97a +size 1153786 diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d2e2a2f343d95d61d4b726df09e00463fc0f40eb --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3521872760c4883ab429067c704af7d3051524dc89faa08875d6edd55fe3cd1 +size 1187188 diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..66fd93a9f640ca1c15b46c260eec2dcf3fdf4161 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:941ebe96bf32b79e70bb907e73c85a0c662ae695e8c381c1470942ee69da5d32 +size 1158306 diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9bad5794ea91e9238762bb02e5f1cf0263538e2 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a15a18ebb9ab075df26a7f49d825502554eebb85f6ab2d1a5919a43a6ad919 +size 1170367 diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7221162175b18ffe53fdb7ee32aee12d5e04fd7 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f55a68bbcaedd47cf64e6b3feec8facbf88d1b4d6a0e5cad6e5a213a85cd27dd +size 1166605 diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a18248c6a829fd840d45524183955a2b1d332601 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log @@ -0,0 +1,308 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736042979.409761]: resetting environment +[INFO] [1736042979.412829]: cleared current plan +[INFO] [1736042979.413755]: moving to home +[INFO] [1736042980.416837]: reached home +[INFO] [1736042980.419118]: reset action history +[INFO] [1736042983.127990]: environment reset +Starting episode 0 +[INFO] [1736042983.129892]: resetting environment +[INFO] [1736042983.132512]: cleared current plan +[INFO] [1736042983.133454]: moving to home +[INFO] [1736042984.135963]: reached home +[INFO] [1736042984.137107]: reset action history +[INFO] [1736042986.847226]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736042994.400733]: resetting environment +[INFO] [1736042994.403663]: cleared current plan +[INFO] [1736042994.404576]: moving to home +[INFO] [1736042995.408033]: reached home +[INFO] [1736042995.409734]: reset action history +[INFO] [1736042998.117532]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/episode_rosbags/episode_0_2025-01-04-21-09-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2022276.log +started roslaunch server http://158.130.50.37:35441/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2022564] +process[print_text-2]: started with pid [2022565] +process[rosbag_record-3]: started with pid [2022566] +started bagging! +For topic gripper_width: timestamp difference is 107182873 for nearest: 1736043011151509178 - target: 1736043011044326305 at idx 170 +[INFO] [1736043025.157383]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736043025.898211]: Stored episode 1. +[INFO] [1736043025.899008]: Saving video... +[INFO] [1736043026.100935]: Video saved! +Starting episode 1 +[INFO] [1736043026.121232]: resetting environment +[INFO] [1736043026.128005]: cleared current plan +[INFO] [1736043026.128341]: moving to home +[INFO] [1736043029.731204]: reached home +[INFO] [1736043029.731709]: reset action history +[INFO] [1736043032.438556]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043036.117617]: resetting environment +[INFO] [1736043036.119919]: cleared current plan +[INFO] [1736043036.120069]: moving to home +[INFO] [1736043037.121269]: reached home +[INFO] [1736043037.121670]: reset action history +[INFO] [1736043039.828991]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/episode_rosbags/episode_1_2025-01-04-21-10-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2022276.log +started roslaunch server http://158.130.50.37:34949/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2022702] +process[print_text-5]: started with pid [2022703] +process[rosbag_record-6]: started with pid [2022727] +started bagging! +For topic gripper_width: timestamp difference is 100532859 for nearest: 1736043044018190234 - target: 1736043043917657375 at idx 56 +For topic gripper_width: timestamp difference is 100730642 for nearest: 1736043044218225702 - target: 1736043044117495060 at idx 58 +For topic gripper_width: timestamp difference is -261397426 for nearest: 1736043049651509459 - target: 1736043049912906885 at idx 135 +For topic gripper_width: timestamp difference is -259465052 for nearest: 1736043052051555322 - target: 1736043052311020374 at idx 166 +For topic gripper_width: timestamp difference is 112630297 for nearest: 1736043059018222023 - target: 1736043058905591726 at idx 249 +[INFO] [1736043066.773252]: Storing episode... +[pub_text-4] killing on exit +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[INFO] [1736043067.571137]: Stored episode 2. +[INFO] [1736043067.571439]: Saving video... +[INFO] [1736043068.216331]: Video saved! +Starting episode 2 +[INFO] [1736043068.244632]: resetting environment +[INFO] [1736043068.257712]: cleared current plan +[INFO] [1736043068.258615]: moving to home +[WARN] [1736043067.936767]: Plan exhausted +[WARN] [1736043067.976625]: Plan exhausted +[WARN] [1736043068.016784]: Plan exhausted +[WARN] [1736043068.056672]: Plan exhausted +[WARN] [1736043068.098577]: Plan exhausted +[WARN] [1736043068.137052]: Plan exhausted +[WARN] [1736043068.176594]: Plan exhausted +[WARN] [1736043068.218160]: Plan exhausted +[WARN] [1736043068.256627]: Plan exhausted +[INFO] [1736043071.861506]: reached home +[INFO] [1736043071.864649]: reset action history +[INFO] [1736043074.595918]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +Invalid key press, try again +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043086.907067]: resetting environment +[INFO] [1736043086.914697]: cleared current plan +[INFO] [1736043086.916004]: moving to home +[INFO] [1736043087.917246]: reached home +[INFO] [1736043087.918437]: reset action history +[INFO] [1736043090.625886]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/episode_rosbags/episode_2_2025-01-04-21-11-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2022276.log +started roslaunch server http://158.130.50.37:33679/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2022862] +process[print_text-8]: started with pid [2022863] +process[rosbag_record-9]: started with pid [2022887] +started bagging! +For topic gripper_width: timestamp difference is 120018816 for nearest: 1736043104418225384 - target: 1736043104298206568 at idx 194 +For topic gripper_width: timestamp difference is 118169507 for nearest: 1736043107218178233 - target: 1736043107100008726 at idx 231 +[INFO] [1736043117.596869]: Storing episode... +[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[INFO] [1736043118.369294]: Stored episode 3. +[INFO] [1736043118.372543]: Saving video... +[INFO] [1736043118.877568]: Video saved! +Starting episode 3 +[INFO] [1736043118.935937]: resetting environment +[INFO] [1736043118.953314]: cleared current plan +[INFO] [1736043118.957613]: moving to home +[WARN] [1736043118.936858]: Plan exhausted +[INFO] [1736043122.563670]: reached home +[INFO] [1736043122.566085]: reset action history +[INFO] [1736043125.293410]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043127.746158]: resetting environment +[INFO] [1736043127.759929]: cleared current plan +[INFO] [1736043127.763799]: moving to home +[INFO] [1736043128.768790]: reached home +[INFO] [1736043128.769494]: reset action history +[INFO] [1736043131.494004]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/episode_rosbags/episode_3_2025-01-04-21-12-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2022276.log +started roslaunch server http://158.130.50.37:44055/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2023019] +process[print_text-11]: started with pid [2023020] +process[rosbag_record-12]: started with pid [2023021] +started bagging! +For topic gripper_width: timestamp difference is 108314996 for nearest: 1736043136218222385 - target: 1736043136109907389 at idx 65 +For topic gripper_width: timestamp difference is 109963301 for nearest: 1736043138218259695 - target: 1736043138108296394 at idx 96 +For topic gripper_width: timestamp difference is 110914710 for nearest: 1736043139418246510 - target: 1736043139307331800 at idx 113 +For topic gripper_width: timestamp difference is 112238056 for nearest: 1736043141018285877 - target: 1736043140906047821 at idx 135 +For topic gripper_width: timestamp difference is -175734879 for nearest: 1736043155751555799 - target: 1736043155927290678 at idx 249 +[INFO] [1736043158.595762]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736043159.412308]: Stored episode 4. +[INFO] [1736043159.417144]: Saving video... +[WARN] [1736043159.938837]: Plan exhausted +[WARN] [1736043159.977560]: Plan exhausted +[WARN] [1736043160.021567]: Plan exhausted +[INFO] [1736043160.074590]: Video saved! +Starting episode 4 +[INFO] [1736043160.128474]: resetting environment +[INFO] [1736043160.149923]: cleared current plan +[INFO] [1736043160.150179]: moving to home +[WARN] [1736043160.062708]: Plan exhausted +[WARN] [1736043160.096844]: Plan exhausted +[WARN] [1736043160.136576]: Plan exhausted +[INFO] [1736043163.752708]: reached home +[INFO] [1736043163.759438]: reset action history +[INFO] [1736043166.550001]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043173.012400]: resetting environment +[INFO] [1736043173.034073]: cleared current plan +[INFO] [1736043173.035871]: moving to home +[INFO] [1736043174.038461]: reached home +[INFO] [1736043174.039818]: reset action history +[INFO] [1736043176.766666]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/episode_rosbags/episode_4_2025-01-04-21-12-56.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2022276.log +started roslaunch server http://158.130.50.37:43077/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2023156] +process[print_text-14]: started with pid [2023157] +process[rosbag_record-15]: started with pid [2023158] +started bagging! +For topic gripper_width: timestamp difference is 109328480 for nearest: 1736043178718177529 - target: 1736043178608849049 at idx 26 +For topic gripper_width: timestamp difference is 113205813 for nearest: 1736043183518221758 - target: 1736043183405015945 at idx 87 +For topic gripper_width: timestamp difference is 113910162 for nearest: 1736043184318286860 - target: 1736043184204376698 at idx 99 +For topic gripper_width: timestamp difference is 121405632 for nearest: 1736043193718222172 - target: 1736043193596816540 at idx 228 +For topic gripper_width: timestamp difference is 126611523 for nearest: 1736043200118221812 - target: 1736043199991610289 at idx 249 +For topic gripper_width: timestamp difference is 129100405 for nearest: 1736043203118257128 - target: 1736043202989156723 at idx 249 +[INFO] [1736043203.885615]: Storing episode... +[pub_text-13] killing on exit +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1736043204.698830]: Stored episode 5. +[INFO] [1736043204.710067]: Saving video... +[INFO] [1736043205.350964]: Video saved! +[INFO] [1736043205.401891]: resetting environment +[INFO] [1736043205.412683]: cleared current plan +[INFO] [1736043205.414241]: moving to home +[WARN] [1736043205.017824]: Plan exhausted +[WARN] [1736043205.059400]: Plan exhausted +[WARN] [1736043205.099853]: Plan exhausted +[WARN] [1736043205.136594]: Plan exhausted +[WARN] [1736043205.176791]: Plan exhausted +[WARN] [1736043205.219079]: Plan exhausted +[WARN] [1736043205.257136]: Plan exhausted +[WARN] [1736043205.297275]: Plan exhausted +[WARN] [1736043205.339170]: Plan exhausted +[WARN] [1736043205.377585]: Plan exhausted +[INFO] [1736043209.021378]: reached home +[INFO] [1736043209.033756]: reset action history +[INFO] [1736043211.872243]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-metadata.json b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bbefde5d1e193a563dd1affbd33a0ddbe2a8f2d0 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:09:31.225064", + "startedAt": "2025-01-05T02:09:30.827805", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3750.0071249999996, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.971, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 650.2632446289062 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fd7978923317586932d238cb170c44032befe5ef --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "258e75c4654817391a5f92df0dfa08592a335185821d354a0825c0235cede97a", "size": 1153786, "path": "media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4"}, "global_step": 663, "_timestamp": 1736043219.574531, "_runtime": 248.7332000732422, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "e3521872760c4883ab429067c704af7d3051524dc89faa08875d6edd55fe3cd1", "size": 1187188, "path": "media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "941ebe96bf32b79e70bb907e73c85a0c662ae695e8c381c1470942ee69da5d32", "size": 1158306, "path": "media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "14a15a18ebb9ab075df26a7f49d825502554eebb85f6ab2d1a5919a43a6ad919", "size": 1170367, "path": "media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "f55a68bbcaedd47cf64e6b3feec8facbf88d1b4d6a0e5cad6e5a213a85cd27dd", "size": 1166605, "path": "media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://qsoa99odzgavr2l8lqjnhgheir20ud7a8a7n6zev3vq4bc2o2gb8m5vv7w8a3nr9unaa7mwlarphk3tpkwq4h3mntno0rd34dwfvogxgkzvm3df8ku4j6o7yuuqurvnz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ain86wk6s2p23djf4rf1mc6lgmsqqx5dw8t384hrqe9f6de22rqds08ufglodkd17iqm90rl8uxmq3askkkdccho38obibmvfqtfy49c39h73zys025svnw3pr171nsr:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 38, "total_num_episodes": 45, "total_success_rate": 0.8444444444444444, "total_success_rate_lb_0.01": 0.7056096195763404, "total_success_rate_ub_0.01": 0.9374039321263025, "total_success_rate_lb_0.025": 0.7111262380654559, "total_success_rate_ub_0.025": 0.9245381168172053, "total_success_rate_lb_0.05": 0.729622602416844, "total_success_rate_ub_0.05": 0.9189273825940081, "total_success_rate_lb_0.1": 0.7748437188016042, "total_success_rate_ub_0.1": 0.9072100165416974, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "d64eacc3b562bec6342e13f63446dbb5bf724740c99a9485dee1a0bf9de97bee", "size": 519, "artifact_path": "wandb-client-artifact://qd13dkm6ma3zzmm5w1wztcsrvpzd6lo5wud2ec5m0fhgtfcz79zd66f4twka3fx59rcrsyfrh82yjzu5xd4qt0a62qf5kh62tg3bxnvgn8e94fw88ww2xe6wpd2k8sbk/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://biohmvwfdzvxw41oq1zc6o7oou52rb335zbqeoo3jn8py67omwqy7hdohrp4yhxgteqfwkez4acwng0s9nhfvh6hy5vkn6m2qwknjj6tq0umyzm7ef33l6o15rvfx49q:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 260}, "eval/total_success_rate": 0.8444444537162781, "eval/total_success_rate_lb_0.01": 0.7056096196174622, "eval/total_success_rate_ub_0.01": 0.9374039173126221, "eval/total_success_rate_lb_0.025": 0.7111262083053589, "eval/total_success_rate_ub_0.025": 0.9245381355285645, "eval/total_success_rate_lb_0.05": 0.7296226024627686, "eval/total_success_rate_ub_0.05": 0.9189273715019226, "eval/total_success_rate_lb_0.1": 0.774843692779541, "eval/total_success_rate_ub_0.1": 0.9072099924087524} \ No newline at end of file diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug-internal.log b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2c16f2d87e44328491af332c774d52c0475f66c6 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug-internal.log @@ -0,0 +1,770 @@ +2025-01-04 21:09:30,841 INFO StreamThr :2022441 [internal.py:wandb_internal():85] W&B internal server running at pid: 2022441, started at: 2025-01-04 21:09:30.840683 +2025-01-04 21:09:30,841 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:09:30,853 INFO WriterThread:2022441 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/run-ot5wit9y.wandb +2025-01-04 21:09:30,854 DEBUG SenderThread:2022441 [sender.py:send():379] send: header +2025-01-04 21:09:30,865 DEBUG SenderThread:2022441 [sender.py:send():379] send: run +2025-01-04 21:09:31,115 INFO SenderThread:2022441 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files +2025-01-04 21:09:31,115 INFO SenderThread:2022441 [sender.py:_start_run_threads():1188] run started: ot5wit9y with start time 1736042970.841331 +2025-01-04 21:09:31,132 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:09:31,132 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:09:31,207 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:09:31,212 DEBUG HandlerThread:2022441 [system_info.py:__init__():26] System info init +2025-01-04 21:09:31,212 DEBUG HandlerThread:2022441 [system_info.py:__init__():41] System info init done +2025-01-04 21:09:31,212 INFO HandlerThread:2022441 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:09:31,212 INFO SystemMonitor:2022441 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:09:31,212 INFO HandlerThread:2022441 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:09:31,213 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:09:31,213 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:09:31,213 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:09:31,214 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:09:31,215 INFO SystemMonitor:2022441 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:09:31,224 DEBUG HandlerThread:2022441 [system_info.py:probe():152] Probing system +2025-01-04 21:09:31,226 DEBUG HandlerThread:2022441 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:09:31,232 DEBUG HandlerThread:2022441 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:09:31,232 DEBUG HandlerThread:2022441 [system_info.py:probe():200] Probing system done +2025-01-04 21:09:31,232 DEBUG HandlerThread:2022441 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:09:31.225064', 'startedAt': '2025-01-05T02:09:30.827805', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.0071249999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.971, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 650.2632446289062}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:09:31,233 INFO HandlerThread:2022441 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:09:31,233 INFO HandlerThread:2022441 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:09:31,233 DEBUG HandlerThread:2022441 [system_info.py:_save_code():44] Saving code +2025-01-04 21:09:31,234 DEBUG HandlerThread:2022441 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:09:31,234 DEBUG HandlerThread:2022441 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:09:31,297 DEBUG HandlerThread:2022441 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:09:31,297 INFO HandlerThread:2022441 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:09:31,300 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:09:31,300 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:09:31,301 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:09:31,301 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:09:31,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:09:31,535 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:09:31,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:31,537 DEBUG SenderThread:2022441 [sender.py:send():379] send: telemetry +2025-01-04 21:09:31,544 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/s90pkjn7-wandb-metadata.json +2025-01-04 21:09:31,571 INFO wandb-upload_2:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/fnuvzh0w-diff.patch +2025-01-04 21:09:31,577 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:09:31,577 INFO HandlerThread:2022441 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/tb" + save: true +} +control { + relay_id: "0e53d899c52a40d3991c399e0b797cce" +} +_info { + stream_id: "ot5wit9y" +} + +2025-01-04 21:09:31,577 DEBUG HandlerThread:2022441 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:09:31,578 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:09:31,580 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/k7l7ltps-code/FISH/eval_robot.py +2025-01-04 21:09:31,738 DEBUG SenderThread:2022441 [sender.py:send():379] send: tbrecord +2025-01-04 21:09:31,738 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:09:31,739 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 with policy live +2025-01-04 21:09:31,925 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/vieyoom2-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH/eval_robot.py +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-metadata.json +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/diff.patch +2025-01-04 21:09:32,117 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code +2025-01-04 21:09:32,118 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH +2025-01-04 21:09:32,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:33,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:34,116 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:34,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:35,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:36,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:36,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:37,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:38,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:39,409 DEBUG SenderThread:2022441 [sender.py:send():379] send: config +2025-01-04 21:09:39,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:40,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:41,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:41,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:42,118 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:42,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:43,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:44,119 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:44,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:45,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:46,120 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:46,535 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:09:46,535 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:09:46,577 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:46,597 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:47,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:48,120 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:48,537 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:49,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:50,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:51,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:52,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:52,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:53,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:54,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:55,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:56,122 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:56,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:57,536 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:09:57,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:58,123 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:09:58,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:09:59,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:00,123 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:00,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:01,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:01,540 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:01,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:02,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:02,660 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:03,124 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml +2025-01-04 21:10:03,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:04,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:05,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:06,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:07,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:08,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:08,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:09,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:10,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:11,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:12,127 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:12,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:13,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:13,541 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:14,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:15,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:16,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:16,538 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:16,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:17,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:18,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:18,599 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:19,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:20,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:21,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:22,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:23,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:24,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:24,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:25,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:26,117 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:10:26,117 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 with policy now +2025-01-04 21:10:26,117 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:10:26,118 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:10:26,118 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:10:26,119 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:10:26,130 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:26,130 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:10:26,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media +2025-01-04 21:10:26,355 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/6avqu09j-media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:10:26,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:26,634 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/ul4148xg-media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:10:27,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:28,131 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:28,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:29,540 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:29,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:30,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:31,215 DEBUG SystemMonitor:2022441 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:10:31,216 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:10:31,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:31,538 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:31,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:32,132 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:32,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:33,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:34,133 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:34,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:35,482 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:35,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:36,134 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:10:36,372 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/wvlxpzyv-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:10:36,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:37,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:38,134 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:38,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:39,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:40,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:41,075 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:41,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:42,136 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:42,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:43,546 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:44,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:45,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:46,137 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:46,218 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:46,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:10:46,539 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:10:46,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:47,547 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:48,549 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:49,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:50,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:51,538 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:51,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:52,139 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:52,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:53,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:54,140 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:10:54,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:55,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:56,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:10:56,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:57,552 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:58,553 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:10:59,553 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:00,142 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:00,553 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:01,219 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:11:01,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:01,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:01,540 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:01,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:02,554 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:03,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:04,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:05,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:06,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:06,773 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:07,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:08,146 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:08,240 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:11:08,240 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 with policy now +2025-01-04 21:11:08,242 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:11:08,243 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:11:08,243 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:11:08,243 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:11:08,461 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/0ykbl0r1-media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:11:08,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:09,146 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:11:09,147 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:11:09,147 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:11:09,411 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/w54k1dwn-media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:11:09,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:10,147 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:10,556 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:11,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:11,863 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:12,556 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:13,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:14,148 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:14,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:15,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:16,149 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:16,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:16,539 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:16,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:17,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:17,698 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:18,557 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:19,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:20,150 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:20,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:21,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:22,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:23,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:23,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:24,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:25,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:26,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:27,152 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:11:27,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:27,996 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:11:27,996 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:11:27,996 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:11:28,153 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:11:28,153 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:28,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:28,997 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:29,559 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:30,154 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:30,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:31,217 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:11:31,539 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:31,539 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:31,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:32,154 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:32,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:33,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:34,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:34,625 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:35,560 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:36,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:37,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:38,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:39,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:40,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:40,561 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:41,562 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:42,562 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:43,571 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:44,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:45,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:46,159 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:46,461 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:46,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:11:46,542 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:11:46,589 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:47,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:48,160 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:11:48,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:49,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:50,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:51,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:51,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:52,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:53,573 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:54,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:55,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:56,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:57,542 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:11:57,575 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:58,577 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:11:58,925 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:11:58,925 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 with policy now +2025-01-04 21:11:58,926 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:11:58,928 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:11:58,928 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:11:58,928 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:11:59,156 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/iato1nkl-media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:11:59,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:11:59,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:11:59,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:11:59,417 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/zipbhv4v-media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:11:59,581 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:00,163 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:00,580 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:01,218 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:12:01,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:01,545 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:01,585 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:02,564 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:02,605 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:03,588 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:04,165 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:04,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:05,589 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:06,165 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:06,594 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:07,599 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:07,727 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:07,822 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:12:07,822 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:12:07,832 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:12:08,166 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:12:08,166 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:12:08,398 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/bn746x97-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:12:08,597 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:09,594 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:10,167 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:10,595 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:11,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:12,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:12,882 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:13,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:14,168 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:14,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:15,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:16,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:16,543 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:16,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:17,590 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:18,170 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:18,218 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:18,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:19,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:20,171 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:20,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:21,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:22,171 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:22,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:23,549 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:23,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:24,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:25,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:26,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:27,591 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:28,550 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:28,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:29,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:30,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:31,219 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:12:31,543 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:31,543 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:31,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:32,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:33,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:33,679 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:34,592 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:35,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:36,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:37,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:38,179 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:38,593 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:38,684 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:39,595 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:40,118 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:12:40,118 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 with policy now +2025-01-04 21:12:40,124 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:12:40,125 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:12:40,125 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:12:40,125 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:12:40,180 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:12:40,180 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:40,181 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:12:40,181 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:12:40,360 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/shzihwmr-media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:12:40,594 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:40,612 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/tc1mvs7u-media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:12:41,603 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:42,180 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:42,602 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:43,606 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:43,754 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:44,607 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:45,608 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:46,182 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:46,548 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:12:46,548 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:12:46,603 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:47,605 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:48,182 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:48,607 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:49,607 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:49,675 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:50,606 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:51,609 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:52,616 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:53,184 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:12:53,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:53,881 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:12:53,881 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:12:53,882 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:12:54,184 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:12:54,185 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:54,610 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:55,085 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:12:55,619 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:56,185 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:56,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:57,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:58,187 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:12:58,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:12:59,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:00,188 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:00,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:00,761 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:01,219 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:13:01,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:01,544 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:01,612 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:02,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:03,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:04,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:05,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:06,190 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:06,361 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:06,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:07,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:08,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:09,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:10,613 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:11,555 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:11,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:12,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:13,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:14,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:15,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:16,193 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:16,544 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:16,545 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:16,614 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:16,667 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:17,615 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:18,615 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:19,616 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:20,618 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:21,617 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:22,195 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:22,556 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:22,617 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:23,618 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:24,196 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:24,626 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:25,391 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:13:25,391 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 with policy now +2025-01-04 21:13:25,401 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:13:25,401 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:25,401 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:25,402 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:25,614 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/2aj43i21-media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:25,620 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:26,197 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval +2025-01-04 21:13:26,470 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/zzouipef-media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:26,633 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:27,572 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:27,632 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:28,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:29,628 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:30,198 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:30,632 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:31,220 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:13:31,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:31,559 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:31,639 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:32,639 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:32,917 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:33,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:34,200 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:34,639 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:35,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:36,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:37,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:38,110 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:38,201 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:38,443 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/f6zo02ej-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:38,635 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:38,936 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:38,937 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:38,937 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:39,065 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:13:39,065 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:13:39,201 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval +2025-01-04 21:13:39,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table +2025-01-04 21:13:39,274 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:13:39,385 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/cpmh7585-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:39,635 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:39,653 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:13:39,692 INFO wandb-upload_1:2022441 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmph02m9aq9 +2025-01-04 21:13:39,697 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:40,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval +2025-01-04 21:13:40,286 INFO SenderThread:2022441 [sender.py:send_request_log_artifact():1525] logged artifact run-ot5wit9y-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0MDYxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzY5NA==', 'latestArtifact': None}} +2025-01-04 21:13:40,286 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:13:40,286 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:13:40,286 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:40,286 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,287 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,287 DEBUG SenderThread:2022441 [sender.py:send():379] send: telemetry +2025-01-04 21:13:40,287 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,291 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,291 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,291 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,292 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,292 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,292 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,292 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,293 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,293 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,293 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,293 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,294 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,294 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,294 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,294 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,295 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,295 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,295 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,295 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,296 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:40,296 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:40,296 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:13:40,466 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/gyjh6vov-media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:40,563 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/s873ui01-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:40,635 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:40,656 INFO wandb-upload_2:2022441 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpl783p45p +2025-01-04 21:13:41,085 INFO SenderThread:2022441 [sender.py:send_request_log_artifact():1525] logged artifact run-ot5wit9y-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0MDYzMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MzY5Ng==', 'latestArtifact': None}} +2025-01-04 21:13:41,085 DEBUG SenderThread:2022441 [sender.py:send():379] send: files +2025-01-04 21:13:41,085 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json with policy now +2025-01-04 21:13:41,085 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:41,085 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:41,086 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:41,202 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:41,354 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/1qjb1onb-media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:41,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:42,203 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:42,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:43,558 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:43,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:44,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:45,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:46,545 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:13:46,545 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:13:46,636 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:47,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:48,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:48,642 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:49,206 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml +2025-01-04 21:13:49,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:50,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:51,637 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:51,741 DEBUG SenderThread:2022441 [sender.py:send():379] send: exit +2025-01-04 21:13:51,741 INFO SenderThread:2022441 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:13:51,742 INFO SenderThread:2022441 [sender.py:send_exit():588] handling runtime: 260 +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:send_exit():594] send defer +2025-01-04 21:13:51,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,743 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:13:51,743 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:13:51,743 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:13:51,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,743 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:13:51,744 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,744 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:13:51,744 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:13:51,744 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:13:51,744 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:13:51,744 DEBUG SystemMonitor:2022441 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:13:51,744 DEBUG SystemMonitor:2022441 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:13:51,798 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:13:51,798 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:13:51,798 INFO HandlerThread:2022441 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:13:51,798 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:13:51,799 DEBUG SenderThread:2022441 [sender.py:send():379] send: stats +2025-01-04 21:13:51,799 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,799 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:13:51,799 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:13:51,799 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:13:51,799 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:51,799 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:13:52,207 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:54,208 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:57,943 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:57,943 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,943 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:57,943 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:13:57,944 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:13:57,944 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:13:57,946 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:57,946 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:57,946 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:13:57,947 DEBUG SenderThread:2022441 [sender.py:send():379] send: history +2025-01-04 21:13:57,947 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:13:57,948 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:57,948 DEBUG SenderThread:2022441 [sender.py:send():379] send: summary +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:13:57,952 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:13:57,952 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:57,952 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:13:57,952 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:13:57,952 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:13:57,952 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:13:57,953 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:57,953 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:13:57,953 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:57,953 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:13:58,105 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:13:58,105 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:58,105 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:13:58,105 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:58,105 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:13:58,105 INFO SenderThread:2022441 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:13:58,108 INFO SenderThread:2022441 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:13:58,110 INFO SenderThread:2022441 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:13:58,119 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:13:58,119 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:58,119 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:13:58,120 DEBUG SenderThread:2022441 [sender.py:send():379] send: artifact +2025-01-04 21:13:58,210 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:58,210 INFO Thread-12 :2022441 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:58,282 INFO SenderThread:2022441 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:13:58,282 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:58,282 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:13:58,282 INFO SenderThread:2022441 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:13:58,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,210 INFO SenderThread:2022441 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml config.yaml +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json wandb-summary.json +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt requirements.txt +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log output.log +2025-01-04 21:13:59,211 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/diff.patch diff.patch +2025-01-04 21:13:59,214 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 media/videos/eval/1_eval_1_e3521872760c4883ab42.mp4 +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 media/videos/eval/2_eval_2_941ebe96bf32b79e70bb.mp4 +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 media/videos/eval/0_eval_0_258e75c4654817391a5f.mp4 +2025-01-04 21:13:59,215 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 media/videos/eval/4_eval_4_f55a68bbcaedd47cf64e.mp4 +2025-01-04 21:13:59,216 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 media/videos/eval/3_eval_3_14a15a18ebb9ab075df2.mp4 +2025-01-04 21:13:59,219 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json media/table/eval/total_success_rate_ci_6_d64eacc3b562bec6342e.table.json +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:13:59,220 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,220 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,220 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:13:59,220 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:13:59,220 INFO SenderThread:2022441 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:13:59,393 INFO wandb-upload_1:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/wandb-summary.json +2025-01-04 21:13:59,401 INFO wandb-upload_2:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/config.yaml +2025-01-04 21:13:59,429 INFO wandb-upload_4:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/output.log +2025-01-04 21:13:59,443 INFO wandb-upload_3:2022441 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/files/requirements.txt +2025-01-04 21:13:59,467 INFO wandb-upload_0:2022441 [upload_job.py:push():130] Uploaded file /tmp/tmpzhh42qunwandb/y0qhm7vn-events.out.tfevents.1736042971.leonmkim-ROG-Strix-G15CS-G15CS.2022276.0 +2025-01-04 21:13:59,667 INFO Thread-11 (_thread_body):2022441 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:13:59,668 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,668 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:13:59,668 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,668 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:13:59,668 INFO SenderThread:2022441 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:13:59,669 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:13:59,669 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,669 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:13:59,670 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,670 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:13:59,670 INFO SenderThread:2022441 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:13:59,743 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,888 INFO SenderThread:2022441 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:13:59,888 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:13:59,888 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,889 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,889 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,889 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:13:59,889 INFO SenderThread:2022441 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:13:59,889 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:13:59,889 INFO HandlerThread:2022441 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send():379] send: final +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send():379] send: footer +2025-01-04 21:13:59,889 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: defer +2025-01-04 21:13:59,890 INFO SenderThread:2022441 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:13:59,890 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,890 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,890 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:13:59,891 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:13:59,892 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:13:59,892 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:13:59,893 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:13:59,893 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:13:59,952 DEBUG SenderThread:2022441 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:13:59,953 INFO MainThread:2022441 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:13:59,953 INFO MainThread:2022441 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:13:59,953 INFO MainThread:2022441 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:13:59,953 DEBUG HandlerThread:2022441 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:13:59,954 INFO HandlerThread:2022441 [handler.py:finish():882] shutting down handler +2025-01-04 21:14:00,893 INFO WriterThread:2022441 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/run-ot5wit9y.wandb +2025-01-04 21:14:00,953 INFO SenderThread:2022441 [sender.py:finish():1615] shutting down sender +2025-01-04 21:14:00,953 INFO SenderThread:2022441 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:14:00,953 INFO SenderThread:2022441 [file_pusher.py:join():175] waiting for file pusher diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug.log b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..af6bce55aa0ee2a611768b6851cc5f5b49ac3901 --- /dev/null +++ b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:09:30,833 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Configure stats pid to 2022276 +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/settings +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug.log +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/logs/debug-internal.log +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y'} +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():619] starting backend +2025-01-04 21:09:30,834 INFO MainThread:2022276 [wandb_init.py:init():623] setting up manager +2025-01-04 21:09:30,839 INFO MainThread:2022276 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:09:30,840 INFO MainThread:2022276 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:09:30,853 INFO MainThread:2022276 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:09:30,862 INFO MainThread:2022276 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:09:31,131 INFO MainThread:2022276 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:09:31,201 INFO MainThread:2022276 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:09:31,201 INFO MainThread:2022276 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:09:31,535 INFO MainThread:2022276 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:09:31,535 INFO MainThread:2022276 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:09:31,536 INFO MainThread:2022276 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:09:31,536 INFO MainThread:2022276 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:09:31,537 INFO MainThread:2022276 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:09:31,538 INFO MainThread:2022276 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ot5wit9y/tb, True +2025-01-04 21:09:39,408 INFO MainThread:2022276 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:14:00,954 WARNING MsgRouterThr:2022276 [router.py:message_loop():77] message_loop has been closed diff --git a/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/run-ot5wit9y.wandb b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/run-ot5wit9y.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e66223f9c6d3aa9eb4e63345d4582239b99401dd Binary files /dev/null and b/ot5wit9y/wandb/run-20250104_210930-ot5wit9y/run-ot5wit9y.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8903c6acba8696f0b4d8194f60506f256ff1709 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e320ecb4cfa486291f1408e935446663e59837e0f97a7c6e1ce50d7aadbf458f +size 910102934 diff --git a/snapshot_11999.pt b/snapshot_11999.pt new file mode 100644 index 0000000000000000000000000000000000000000..38fd2360fb0852a894537d9415abf8b1ed32feb9 --- /dev/null +++ b/snapshot_11999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de90195281756de69f907a9e3b5db0b00bc3f7bda09e66d548f922c4fad07549 +size 910102934 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..91eacf69361980a1f51cc71fb96c6b72545f88e5 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2105ee0c3c63ab90086630b2c44475356e000677d764fcae03bf83a4cd98f5ec +size 910102934 diff --git a/snapshot_13999.pt b/snapshot_13999.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e75f334e23bd2c1a4999154e6d19ac0a396f30e --- /dev/null +++ b/snapshot_13999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48c07a5e055435609be8e248da225b443a398a989cf4eea7a8cd6f1fa9a4c696 +size 910102934 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1d3c72ec45619972e60f1c87e7b5f000c6d430d --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c78a997fce17d845ce7cfd30b7921f78780bd366a641228667030cc02af093 +size 910102934 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..41b169c342319d4650da37e857fc5b4e0737e7a3 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa3fee9c7e1a63b49a72b4eaa638d73d9b1a7579bdf9baa5c0e666af7439ae5 +size 910102934 diff --git a/snapshot_5999.pt b/snapshot_5999.pt new file mode 100644 index 0000000000000000000000000000000000000000..39d056e37d2f8c82983703386b4d05bba970d112 --- /dev/null +++ b/snapshot_5999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1c6af169fdaa39bc26be28e951d91a5703c448d993c6f10c8a6971f74f2a5ea +size 910102934 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..43e6d25e3492bb7e84d49140bdbaa36d3e8398e6 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c028ff4653788b603b49c811e2ccd94ee33367737406a17ff0d13689eb7f714c +size 910102934 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c916ab42721795bc7e62006086a29241d55c067 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64ba998822ef0df9a435c48afa353fbcfa82876d9f8b487c7a69287fbcefed6e +size 910102934 diff --git a/snapshot_7999.pt b/snapshot_7999.pt new file mode 100644 index 0000000000000000000000000000000000000000..e11ff374a999f392711a7870623728a2a107c988 --- /dev/null +++ b/snapshot_7999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4254c0e412779466a592b34b3d29b39b8d11980535652d36de02d830aed7bd6 +size 910102934 diff --git a/snapshot_8379.pt b/snapshot_8379.pt new file mode 100644 index 0000000000000000000000000000000000000000..e11ff374a999f392711a7870623728a2a107c988 --- /dev/null +++ b/snapshot_8379.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4254c0e412779466a592b34b3d29b39b8d11980535652d36de02d830aed7bd6 +size 910102934 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..9dde8329a74bba54b762597b001762d50a5f38b2 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4070dee5efc38ea2dbc6b93ad32159afefaac5e6650ad9464bd76ab8884d400d +size 910102934 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1d3c72ec45619972e60f1c87e7b5f000c6d430d --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c78a997fce17d845ce7cfd30b7921f78780bd366a641228667030cc02af093 +size 910102934 diff --git a/ukc5qfsm/.hydra/config.yaml b/ukc5qfsm/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/ukc5qfsm/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ukc5qfsm/.hydra/hydra.yaml b/ukc5qfsm/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7b36d71519f8af0ad9b630ff084bdf7c951350f --- /dev/null +++ b/ukc5qfsm/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ukc5qfsm/.hydra/overrides.yaml b/ukc5qfsm/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ukc5qfsm/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ukc5qfsm/episode_rosbags/aligned_depth_to_color_K.npy b/ukc5qfsm/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ukc5qfsm/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ukc5qfsm/episode_rosbags/cam_tf_world.npy b/ukc5qfsm/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/ukc5qfsm/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/ukc5qfsm/episode_rosbags/color_K.npy b/ukc5qfsm/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ukc5qfsm/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ukc5qfsm/episode_rosbags/depth_K.npy b/ukc5qfsm/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/ukc5qfsm/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/ukc5qfsm/eval_robot.log b/ukc5qfsm/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8af3c57eed423b8e90d081501e0b57d23df5087f --- /dev/null +++ b/ukc5qfsm/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:16:06,280][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:16:06,283][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/ukc5qfsm/eval_video/0_eval.mp4 b/ukc5qfsm/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39f496d52ba3ab5063a427110c75ab567d72dd16 --- /dev/null +++ b/ukc5qfsm/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00fe423cbd21a3f93a8f49d18ed86c5d26b1114dc25c17bd2e570a0991bf99fe +size 1133780 diff --git a/ukc5qfsm/eval_video/1_eval.mp4 b/ukc5qfsm/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..82a4783474e6660c66b184358c90f2f21c2477d0 --- /dev/null +++ b/ukc5qfsm/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a6cf8eaa58e555adc3191ff93a0be53b3d45cc0aab69c0a04993c5c0fd6788 +size 1210152 diff --git a/ukc5qfsm/eval_video/2_eval.mp4 b/ukc5qfsm/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb6b9c7403921c6d5402f587e77b7e82a7eb02ae --- /dev/null +++ b/ukc5qfsm/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60ba9e6f07a8e7ecafdb195bc5785362bfa055b68605ae8b6130e7223024372f +size 1207938 diff --git a/ukc5qfsm/eval_video/3_eval.mp4 b/ukc5qfsm/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acd2f45dd305f2d118446818722d554da78b24df --- /dev/null +++ b/ukc5qfsm/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e5592f1dae1997ff968a60f1e3ba8250e00bfcb826bbdd9c7bf4480924e0788 +size 1152050 diff --git a/ukc5qfsm/eval_video/4_eval.mp4 b/ukc5qfsm/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c98f47dd3207e05bea22fc0d9795c0a20c810524 --- /dev/null +++ b/ukc5qfsm/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04fddf9f1781ca8c63e3cc9c73122b5e0a57d817663ee076dd71f11d71a3bc36 +size 1259877 diff --git a/ukc5qfsm/tb/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 b/ukc5qfsm/tb/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 new file mode 100644 index 0000000000000000000000000000000000000000..3fa2c9caf3a477737dfbd047cc180f928f431073 --- /dev/null +++ b/ukc5qfsm/tb/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90488ccc479800d503f949b37a026a31af884447b349575c01e176446e892c05 +size 6539 diff --git a/ukc5qfsm/wandb/debug-internal.log b/ukc5qfsm/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d2116b3dd360d5bfef7cc146d54d7225f9d76ff5 --- /dev/null +++ b/ukc5qfsm/wandb/debug-internal.log @@ -0,0 +1,810 @@ +2025-01-03 22:16:12,663 INFO StreamThr :1981451 [internal.py:wandb_internal():85] W&B internal server running at pid: 1981451, started at: 2025-01-03 22:16:12.662934 +2025-01-03 22:16:12,664 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:16:12,676 INFO WriterThread:1981451 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/run-ukc5qfsm.wandb +2025-01-03 22:16:12,677 DEBUG SenderThread:1981451 [sender.py:send():379] send: header +2025-01-03 22:16:12,684 DEBUG SenderThread:1981451 [sender.py:send():379] send: run +2025-01-03 22:16:12,928 INFO SenderThread:1981451 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files +2025-01-03 22:16:12,928 INFO SenderThread:1981451 [sender.py:_start_run_threads():1188] run started: ukc5qfsm with start time 1735960572.663657 +2025-01-03 22:16:12,943 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:16:12,944 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:16:12,988 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:16:12,993 DEBUG HandlerThread:1981451 [system_info.py:__init__():26] System info init +2025-01-03 22:16:12,993 DEBUG HandlerThread:1981451 [system_info.py:__init__():41] System info init done +2025-01-03 22:16:12,993 INFO HandlerThread:1981451 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:16:12,993 INFO SystemMonitor:1981451 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:16:12,993 INFO HandlerThread:1981451 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:16:12,994 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:16:12,994 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:16:12,995 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:16:12,995 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:16:12,996 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:16:13,004 DEBUG HandlerThread:1981451 [system_info.py:probe():152] Probing system +2025-01-03 22:16:13,005 DEBUG HandlerThread:1981451 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_info.py:probe():200] Probing system done +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:16:13.004213', 'startedAt': '2025-01-04T03:16:12.653740', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.011375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.972, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 831.5409240722656}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:16:13,010 INFO HandlerThread:1981451 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:16:13,010 INFO HandlerThread:1981451 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_info.py:_save_code():44] Saving code +2025-01-03 22:16:13,012 DEBUG HandlerThread:1981451 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:16:13,012 DEBUG HandlerThread:1981451 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:16:13,076 DEBUG HandlerThread:1981451 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:16:13,076 INFO HandlerThread:1981451 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:16:13,079 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:16:13,079 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:16:13,080 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:16:13,080 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:16:13,305 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/ks1ruya0-wandb-metadata.json +2025-01-03 22:16:13,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:16:13,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:16:13,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:13,319 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:13,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:13,321 INFO HandlerThread:1981451 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/tb" + save: true +} +control { + relay_id: "9d81edaa288d4dcfb7b8eac6a9e912ce" +} +_info { + stream_id: "ukc5qfsm" +} + +2025-01-03 22:16:13,321 DEBUG HandlerThread:1981451 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:16:13,354 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/fej5ou4n-code/FISH/eval_robot.py +2025-01-03 22:16:13,395 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/4rfg9bex-diff.patch +2025-01-03 22:16:13,459 DEBUG SenderThread:1981451 [sender.py:send():379] send: telemetry +2025-01-03 22:16:13,459 DEBUG SenderThread:1981451 [sender.py:send():379] send: tbrecord +2025-01-03 22:16:13,460 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:16:13,460 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 with policy live +2025-01-03 22:16:13,657 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/99uae6cr-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-metadata.json +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH/eval_robot.py +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/diff.patch +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:16:13,931 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH +2025-01-03 22:16:13,931 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code +2025-01-03 22:16:14,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:15,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:15,929 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:16,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:17,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:18,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:18,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:19,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:20,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:21,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:22,109 DEBUG SenderThread:1981451 [sender.py:send():379] send: config +2025-01-03 22:16:22,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:23,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:23,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:23,931 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:24,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:25,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:26,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:27,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:27,933 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:28,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:28,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:28,361 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:28,442 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:29,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:30,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:31,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:31,934 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:32,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:33,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:34,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:34,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:35,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:36,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:37,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:38,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:39,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:40,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:40,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:41,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:42,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:43,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:43,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:43,365 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:44,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:45,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:45,415 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:45,937 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml +2025-01-03 22:16:46,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:47,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:48,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:49,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:49,939 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:50,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:50,725 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:51,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:51,939 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:52,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:53,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:53,940 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:54,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:55,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:56,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:56,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:57,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:58,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:58,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:58,362 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:59,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:00,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:01,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:01,400 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:02,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:03,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:04,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:05,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:06,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:07,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:07,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:08,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:09,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:10,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:11,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:11,945 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:12,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:12,856 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:12,996 DEBUG SystemMonitor:1981451 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:17:12,997 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:17:13,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:13,319 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:13,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:13,945 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:14,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:15,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:16,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:17,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:17,877 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:17,946 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:18,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:19,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:19,947 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:20,329 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:20,397 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:17:20,398 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:17:20,398 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 with policy now +2025-01-03 22:17:20,398 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:17:20,398 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:17:20,399 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:17:20,655 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/28g0bvwa-media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:17:20,947 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos +2025-01-03 22:17:21,274 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/6yk9q299-media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:17:21,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:21,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:22,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:23,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:23,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:24,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:25,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:25,949 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:26,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:27,329 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:27,950 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:28,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:28,319 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:28,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:28,423 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:28,950 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:17:29,200 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/2ev29hu3-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:17:29,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:29,950 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:30,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:31,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:31,951 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:32,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:33,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:33,952 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:34,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:34,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:35,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:35,953 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:36,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:37,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:38,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:39,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:40,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:40,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:41,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:42,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:43,005 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:17:43,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:43,320 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:43,361 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:44,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:45,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:45,956 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:46,273 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:46,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:47,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:48,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:49,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:50,333 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:51,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:51,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:52,349 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:53,349 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:54,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:55,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:56,351 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:56,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:57,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:57,960 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:58,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:58,320 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:58,361 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:59,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:00,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:01,354 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:01,521 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:01,777 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:18:01,777 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 with policy now +2025-01-03 22:18:01,777 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:18:01,778 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:18:01,778 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:18:01,779 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:18:01,961 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:18:01,961 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:01,961 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:18:01,962 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:18:02,044 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/h7qkclf6-media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:18:02,314 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/3uv2omx5-media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:18:02,354 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:03,359 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:03,962 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:04,353 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:05,356 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:06,355 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:06,541 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:07,360 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:07,963 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:08,357 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:09,360 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:09,964 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:10,360 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:11,359 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:12,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:12,369 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:12,998 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:18:13,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:13,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:13,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:14,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:15,367 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:16,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:17,367 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:17,450 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:18,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:19,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:20,367 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:21,369 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:22,370 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:23,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:23,368 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:24,370 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:25,369 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:26,373 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:27,372 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:28,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:28,323 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:28,372 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:28,468 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:29,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:30,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:31,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:32,374 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:32,765 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:18:32,765 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:18:32,765 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:18:32,972 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:18:32,972 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:18:33,216 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/1s94kxc5-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:18:33,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:33,972 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:34,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:34,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:35,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:36,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:37,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:37,974 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:38,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:39,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:39,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:39,974 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:40,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:41,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:42,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:42,998 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:18:43,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:43,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:43,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:44,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:44,438 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:45,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:46,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:47,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:47,977 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:48,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:49,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:49,978 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:50,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:50,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:51,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:52,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:53,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:54,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:55,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:56,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:56,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:57,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:58,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:58,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:58,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:59,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:00,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:01,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:01,444 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:02,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:03,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:04,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:05,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:05,770 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:19:05,770 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 with policy now +2025-01-03 22:19:05,773 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:19:05,774 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:05,774 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:05,774 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:19:06,023 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/8th1ww3u-media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:19:06,270 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/98tn6rf5-media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:19:06,385 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:06,783 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:07,384 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:07,984 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:08,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:09,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:10,393 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:11,386 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:11,985 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:12,267 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:12,385 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:12,999 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:19:13,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:13,322 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:13,387 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:13,986 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:14,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:15,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:15,986 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:16,393 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:16,600 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:16,600 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:16,601 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:16,987 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:16,987 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:17,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:17,987 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:18,021 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:18,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:19,399 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:20,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:21,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:21,988 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:22,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:23,253 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:23,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:24,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:25,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:26,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:27,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:28,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:28,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:28,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:28,410 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:29,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:30,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:31,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:32,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:33,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:34,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:34,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:35,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:36,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:37,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:38,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:39,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:40,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:40,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:41,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:42,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:42,999 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:19:43,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:43,322 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:43,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:44,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:45,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:45,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:46,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:47,406 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:47,997 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:48,069 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:19:48,069 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 with policy now +2025-01-03 22:19:48,072 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:19:48,072 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:48,072 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:48,073 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:48,352 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/5qtclipz-media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:19:48,393 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:48,997 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:48,998 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:19:48,998 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:19:49,287 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/fqbwdbex-media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:19:49,398 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:49,998 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:50,401 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:51,329 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:51,401 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:52,405 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:53,408 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:53,999 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:54,407 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:55,408 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:56,000 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:56,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:56,649 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:57,411 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:58,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:58,326 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:58,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:58,653 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:58,654 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:58,654 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:59,001 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:59,001 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:59,241 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/26qxe994-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:59,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:00,001 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:00,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:01,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:02,002 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:02,269 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:02,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:03,418 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:04,003 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:04,412 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:05,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:06,003 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:06,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:07,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:07,707 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:08,416 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:09,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:10,416 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:11,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:12,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:12,999 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:20:13,000 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:13,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:20:13,323 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:20:13,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:14,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:15,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:16,010 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:16,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:17,422 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:18,117 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:18,415 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:19,415 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:20,424 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:21,424 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:22,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:23,333 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:23,429 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:24,430 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:25,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:26,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:27,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:28,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:20:28,323 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:20:28,419 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:28,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:29,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:30,428 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:31,428 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:32,430 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:33,443 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:33,485 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:33,516 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:20:33,516 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 with policy now +2025-01-03 22:20:33,557 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:20:33,557 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:33,557 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:33,558 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:33,768 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/xqd8w5n6-media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:20:34,299 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/8m3q42gn-media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:20:34,441 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:35,443 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:36,021 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:36,440 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:37,449 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:38,444 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:39,340 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:39,442 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:40,022 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:40,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:41,512 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:41,712 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:41,712 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:41,713 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:41,788 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:20:41,789 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:20:41,984 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval +2025-01-03 22:20:42,204 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/60xfdf6p-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:20:42,317 INFO wandb-upload_2:1981451 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxci8vdsb +2025-01-03 22:20:42,349 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:20:42,359 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:20:42,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:42,716 INFO SenderThread:1981451 [sender.py:send_request_log_artifact():1525] logged artifact run-ukc5qfsm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjMwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTkzNg==', 'latestArtifact': None}} +2025-01-03 22:20:42,716 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:20:42,716 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-03 22:20:42,716 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:42,717 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,717 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,717 DEBUG SenderThread:1981451 [sender.py:send():379] send: telemetry +2025-01-03 22:20:42,717 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,721 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,721 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,721 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,721 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,722 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,722 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,722 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,722 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,723 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,723 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,723 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,723 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,723 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,724 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,724 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,724 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,724 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,725 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,725 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:20:42,980 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/u69p8n3n-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:20:43,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:43,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:20:43,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:20:43,024 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval +2025-01-03 22:20:43,107 INFO wandb-upload_2:1981451 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp420an72j +2025-01-03 22:20:43,226 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/vi5qq3r2-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:20:43,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:20:43,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:43,530 INFO SenderThread:1981451 [sender.py:send_request_log_artifact():1525] logged artifact run-ukc5qfsm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjMxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTk0MQ==', 'latestArtifact': None}} +2025-01-03 22:20:43,531 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:20:43,531 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json with policy now +2025-01-03 22:20:43,531 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:43,531 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:43,532 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:43,532 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:20:43,532 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:20:43,763 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/pj5thlno-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:20:44,024 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:44,024 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:44,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:44,644 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:45,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:46,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:47,513 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:48,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:49,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:50,335 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:50,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:51,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:52,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:52,613 DEBUG SenderThread:1981451 [sender.py:send():379] send: exit +2025-01-03 22:20:52,613 INFO SenderThread:1981451 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:20:52,613 INFO SenderThread:1981451 [sender.py:send_exit():588] handling runtime: 279 +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:send_exit():594] send defer +2025-01-03 22:20:52,614 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,614 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:20:52,614 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:20:52,614 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,614 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:20:52,615 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,615 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:20:52,615 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:20:52,615 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,615 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:20:52,615 INFO HandlerThread:1981451 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:20:52,615 DEBUG SystemMonitor:1981451 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:20:52,615 DEBUG SystemMonitor:1981451 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:20:52,616 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:20:52,616 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:20:52,686 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:20:52,686 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:20:52,687 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:20:52,687 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,687 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:20:52,687 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:20:52,687 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:20:52,687 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,687 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:20:52,688 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,688 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:20:52,688 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:20:52,688 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,688 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:20:53,027 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:54,028 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:58,718 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:20:58,718 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:58,718 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,719 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:20:58,719 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:58,720 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:58,720 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:20:58,720 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:58,720 DEBUG SenderThread:1981451 [sender.py:send():379] send: summary +2025-01-03 22:20:58,724 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:58,724 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,724 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:20:58,724 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:20:58,725 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:58,725 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:20:58,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,725 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:20:58,728 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:58,826 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:20:58,826 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:58,826 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:20:58,826 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,826 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:20:59,030 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:59,030 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml +2025-01-03 22:20:59,614 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:20:59,761 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:20:59,761 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:20:59,761 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:59,761 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:20:59,762 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:59,762 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:20:59,762 INFO SenderThread:1981451 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:20:59,764 INFO SenderThread:1981451 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:20:59,766 INFO SenderThread:1981451 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:20:59,779 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:20:59,780 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:59,780 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:20:59,780 DEBUG SenderThread:1981451 [sender.py:send():379] send: artifact +2025-01-03 22:21:00,030 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:21:00,125 INFO wandb-upload_2:1981451 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyehzxc9p +2025-01-03 22:21:00,125 INFO wandb-upload_1:1981451 [upload_job.py:push():85] Skipped uploading /tmp/tmpnmd_mrlg/wandb-job.json +2025-01-03 22:21:00,342 INFO wandb-upload_0:1981451 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwj6kj7gm +2025-01-03 22:21:00,615 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:21:00,901 INFO SenderThread:1981451 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'versionIndex': 411}}} +2025-01-03 22:21:00,901 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:00,901 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:21:00,902 INFO SenderThread:1981451 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml config.yaml +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json wandb-summary.json +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt requirements.txt +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:21:01,032 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:21:01,032 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log output.log +2025-01-03 22:21:01,032 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/diff.patch diff.patch +2025-01-03 22:21:01,036 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:21:01,040 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:21:01,040 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:21:01,040 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:21:01,040 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:21:01,040 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:21:01,040 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:21:01,041 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:01,041 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:21:01,041 INFO SenderThread:1981451 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:21:01,216 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt +2025-01-03 22:21:01,219 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml +2025-01-03 22:21:01,259 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:21:01,279 INFO wandb-upload_4:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:21:01,326 INFO wandb-upload_3:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/fj4w5byq-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:21:01,527 INFO Thread-11 (_thread_body):1981451 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:21:01,527 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:21:01,527 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:21:01,528 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:01,528 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:21:01,528 INFO SenderThread:1981451 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:21:01,531 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:21:01,531 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:21:01,531 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:21:01,531 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:01,531 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:21:01,531 INFO SenderThread:1981451 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:21:01,615 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:21:06,617 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:11,618 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:16,619 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:21,620 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:26,622 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:31,623 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:36,624 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:41,625 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:46,626 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:51,628 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:56,629 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:22:01,630 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:22:01,657 WARNING FileStreamThread:1981451 [file_stream.py:request_with_retry():674] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/serialexperimentsleon/extrinsic_contact_downstream/ukc5qfsm/file_stream. func: >, args: ('https://api.wandb.ai/files/serialexperimentsleon/extrinsic_contact_downstream/ukc5qfsm/file_stream',), kwargs: {'json': {'complete': False, 'failed': False, 'dropped': 0, 'uploaded': ['wandb-summary.json', 'requirements.frozen.txt', 'diff.patch', 'wandb-job.json', 'config.yaml', 'output.log', 'requirements.txt', 'events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0']}} +2025-01-03 22:22:03,821 INFO SenderThread:1981451 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:22:03,821 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:22:03,821 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:22:03,821 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:22:03,821 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:22:03,821 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:22:03,822 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:22:03,822 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:22:03,822 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send():379] send: final +2025-01-03 22:22:03,822 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send():379] send: footer +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:22:03,822 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:22:03,823 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:22:03,823 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:22:03,823 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:22:03,823 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:22:03,824 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:22:03,825 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:22:03,826 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:22:03,826 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:22:03,906 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:22:03,906 INFO MainThread:1981451 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:22:03,907 INFO MainThread:1981451 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:22:03,907 INFO MainThread:1981451 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:22:03,907 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:22:03,907 INFO HandlerThread:1981451 [handler.py:finish():882] shutting down handler +2025-01-03 22:22:04,824 INFO WriterThread:1981451 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/run-ukc5qfsm.wandb +2025-01-03 22:22:04,906 INFO SenderThread:1981451 [sender.py:finish():1615] shutting down sender +2025-01-03 22:22:04,906 INFO SenderThread:1981451 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:22:04,906 INFO SenderThread:1981451 [file_pusher.py:join():175] waiting for file pusher diff --git a/ukc5qfsm/wandb/debug.log b/ukc5qfsm/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1317f07e40f7c906f31d10f10172420b3e1b3d38 --- /dev/null +++ b/ukc5qfsm/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Configure stats pid to 1981219 +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/settings +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug.log +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug-internal.log +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm'} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():619] starting backend +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():623] setting up manager +2025-01-03 22:16:12,662 INFO MainThread:1981219 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:16:12,663 INFO MainThread:1981219 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:16:12,676 INFO MainThread:1981219 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:16:12,682 INFO MainThread:1981219 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:16:12,943 INFO MainThread:1981219 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:16:12,983 INFO MainThread:1981219 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:16:12,984 INFO MainThread:1981219 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:16:13,318 INFO MainThread:1981219 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:16:13,318 INFO MainThread:1981219 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:16:13,319 INFO MainThread:1981219 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:16:13,319 INFO MainThread:1981219 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:16:13,320 INFO MainThread:1981219 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:16:13,321 INFO MainThread:1981219 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/tb, True +2025-01-03 22:16:22,108 INFO MainThread:1981219 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:22:04,908 WARNING MsgRouterThr:1981219 [router.py:message_loop():77] message_loop has been closed diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH/eval_robot.py b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9830383f3d5cf9f4b0f60cc89fdc7d55f38267b3 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735960572 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/diff.patch b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json new file mode 100644 index 0000000000000000000000000000000000000000..5271efacb3db6a0813497da4611af43307900d95 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 15, 1.0, 0.9928352235762072, 1, 0.7993280141947043, 1, 0.8249338715618673, 1, 1, 1]]} \ No newline at end of file diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39f496d52ba3ab5063a427110c75ab567d72dd16 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00fe423cbd21a3f93a8f49d18ed86c5d26b1114dc25c17bd2e570a0991bf99fe +size 1133780 diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..82a4783474e6660c66b184358c90f2f21c2477d0 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a6cf8eaa58e555adc3191ff93a0be53b3d45cc0aab69c0a04993c5c0fd6788 +size 1210152 diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb6b9c7403921c6d5402f587e77b7e82a7eb02ae --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60ba9e6f07a8e7ecafdb195bc5785362bfa055b68605ae8b6130e7223024372f +size 1207938 diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acd2f45dd305f2d118446818722d554da78b24df --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e5592f1dae1997ff968a60f1e3ba8250e00bfcb826bbdd9c7bf4480924e0788 +size 1152050 diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c98f47dd3207e05bea22fc0d9795c0a20c810524 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04fddf9f1781ca8c63e3cc9c73122b5e0a57d817663ee076dd71f11d71a3bc36 +size 1259877 diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..536f2e2aa3850d63e17eecb4c5c74d723670d375 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log @@ -0,0 +1,273 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735960582.109340]: resetting environment +[INFO] [1735960582.112372]: cleared current plan +[INFO] [1735960582.113372]: moving to home +[INFO] [1735960583.115493]: reached home +[INFO] [1735960583.117187]: reset action history +[INFO] [1735960585.826345]: environment reset +Starting episode 0 +[INFO] [1735960585.828539]: resetting environment +[INFO] [1735960585.831065]: cleared current plan +[INFO] [1735960585.832112]: moving to home +[INFO] [1735960586.834893]: reached home +[INFO] [1735960586.836282]: reset action history +[INFO] [1735960589.546365]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735960608.668638]: resetting environment +[INFO] [1735960608.674486]: cleared current plan +[INFO] [1735960608.675312]: moving to home +[INFO] [1735960609.677426]: reached home +[INFO] [1735960609.678538]: reset action history +[INFO] [1735960612.385784]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/episode_rosbags/episode_0_2025-01-03-22-16-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1981219.log +started roslaunch server http://158.130.50.37:44833/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1981577] +process[print_text-2]: started with pid [1981578] +process[rosbag_record-3]: started with pid [1981579] +started bagging! +For topic gripper_width: timestamp difference is -103445351 for nearest: 1735960628861538828 - target: 1735960628964984179 at idx 237 +For topic gripper_width: timestamp difference is -100552965 for nearest: 1735960632661537480 - target: 1735960632762090445 at idx 249 +For topic gripper_width: timestamp difference is 101297566 for nearest: 1735960635228230902 - target: 1735960635126933336 at idx 249 +For topic gripper_width: timestamp difference is 102090646 for nearest: 1735960636228232194 - target: 1735960636126141548 at idx 249 +For topic gripper_width: timestamp difference is 102830675 for nearest: 1735960637228154447 - target: 1735960637125323772 at idx 249 +For topic gripper_width: timestamp difference is 103377867 for nearest: 1735960637828210640 - target: 1735960637724832773 at idx 249 +[INFO] [1735960639.428060]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735960640.148115]: Stored episode 1. +[INFO] [1735960640.149029]: Saving video... +[INFO] [1735960640.354074]: Video saved! +Starting episode 1 +[INFO] [1735960640.366234]: resetting environment +[INFO] [1735960640.371485]: cleared current plan +[INFO] [1735960640.371776]: moving to home +[INFO] [1735960644.074342]: reached home +[INFO] [1735960644.075960]: reset action history +[INFO] [1735960646.788383]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960648.203681]: resetting environment +[INFO] [1735960648.210331]: cleared current plan +[INFO] [1735960648.210562]: moving to home +[INFO] [1735960649.212553]: reached home +[INFO] [1735960649.212972]: reset action history +[INFO] [1735960651.920018]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/episode_rosbags/episode_1_2025-01-03-22-17-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1981219.log +started roslaunch server http://158.130.50.37:38253/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1981736] +process[print_text-5]: started with pid [1981737] +process[rosbag_record-6]: started with pid [1981738] +started bagging! +For topic gripper_width: timestamp difference is 124720784 for nearest: 1735960664228232594 - target: 1735960664103511810 at idx 181 +For topic gripper_width: timestamp difference is 100348354 for nearest: 1735960675328239799 - target: 1735960675227891445 at idx 249 +[INFO] [1735960680.700124]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1735960681.520322]: Stored episode 2. +[INFO] [1735960681.522759]: Saving video... +[INFO] [1735960681.744805]: Video saved! +Starting episode 2 +[INFO] [1735960681.779620]: resetting environment +[INFO] [1735960681.791650]: cleared current plan +[INFO] [1735960681.791859]: moving to home +[INFO] [1735960685.497719]: reached home +[INFO] [1735960685.502413]: reset action history +[INFO] [1735960688.277467]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960712.015379]: resetting environment +[INFO] [1735960712.023452]: cleared current plan +[INFO] [1735960712.024620]: moving to home +[INFO] [1735960713.027718]: reached home +[INFO] [1735960713.034007]: reset action history +[INFO] [1735960715.810911]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/episode_rosbags/episode_2_2025-01-03-22-18-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1981219.log +started roslaunch server http://158.130.50.37:36789/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1981878] +process[print_text-8]: started with pid [1981879] +process[rosbag_record-9]: started with pid [1981880] +started bagging! +For topic gripper_width: timestamp difference is 108593882 for nearest: 1735960726761537255 - target: 1735960726652943373 at idx 145 +For topic gripper_width: timestamp difference is 109811878 for nearest: 1735960728228206730 - target: 1735960728118394852 at idx 167 +[INFO] [1735960744.741096]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735960745.509707]: Stored episode 3. +[INFO] [1735960745.513417]: Saving video... +[INFO] [1735960745.749022]: Video saved! +Starting episode 3 +[INFO] [1735960745.773494]: resetting environment +[INFO] [1735960745.781614]: cleared current plan +[INFO] [1735960745.782565]: moving to home +[INFO] [1735960749.484982]: reached home +[INFO] [1735960749.488552]: reset action history +[INFO] [1735960752.267038]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960755.942718]: resetting environment +[INFO] [1735960755.960473]: cleared current plan +[INFO] [1735960755.962586]: moving to home +[INFO] [1735960756.973366]: reached home +[INFO] [1735960756.976923]: reset action history +[INFO] [1735960759.756042]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/episode_rosbags/episode_3_2025-01-03-22-19-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1981219.log +started roslaunch server http://158.130.50.37:41437/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1982017] +process[print_text-11]: started with pid [1982018] +process[rosbag_record-12]: started with pid [1982019] +started bagging! +For topic gripper_width: timestamp difference is 123579668 for nearest: 1735960786628203320 - target: 1735960786504623652 at idx 249 +[INFO] [1735960786.997636]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735960787.822223]: Stored episode 4. +[INFO] [1735960787.824085]: Saving video... +[INFO] [1735960788.035079]: Video saved! +Starting episode 4 +[INFO] [1735960788.075848]: resetting environment +[INFO] [1735960788.087083]: cleared current plan +[INFO] [1735960788.088057]: moving to home +[INFO] [1735960791.795556]: reached home +[INFO] [1735960791.801323]: reset action history +[INFO] [1735960794.600751]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960798.440741]: resetting environment +[INFO] [1735960798.454518]: cleared current plan +[INFO] [1735960798.459287]: moving to home +[INFO] [1735960799.475252]: reached home +[INFO] [1735960799.480099]: reset action history +[INFO] [1735960802.269228]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/episode_rosbags/episode_4_2025-01-03-22-20-02.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1981219.log +started roslaunch server http://158.130.50.37:40745/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1982151] +process[print_text-14]: started with pid [1982152] +process[rosbag_record-15]: started with pid [1982153] +started bagging! +For topic gripper_width: timestamp difference is 112785725 for nearest: 1735960814061540989 - target: 1735960813948755264 at idx 144 +[INFO] [1735960832.377007]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735960833.182543]: Stored episode 5. +[INFO] [1735960833.184867]: Saving video... +[INFO] [1735960833.435710]: Video saved! +[INFO] [1735960833.530606]: resetting environment +[INFO] [1735960833.556672]: cleared current plan +[INFO] [1735960833.572216]: moving to home +[INFO] [1735960837.288308]: reached home +[INFO] [1735960837.293256]: reset action history +[INFO] [1735960840.138920]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-metadata.json b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..952ff2b9f19c84ede725dc4a65b762f038fab620 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:16:13.004213", + "startedAt": "2025-01-04T03:16:12.653740", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3375.011375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.972, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.019, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 831.5409240722656 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7b2d12994cd2d3a2f93d2d7d120360f0270dfc01 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "00fe423cbd21a3f93a8f49d18ed86c5d26b1114dc25c17bd2e570a0991bf99fe", "size": 1133780, "path": "media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4"}, "global_step": 665, "_timestamp": 1735960842.2522407, "_runtime": 269.58858370780945, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "e3a6cf8eaa58e555adc3191ff93a0be53b3d45cc0aab69c0a04993c5c0fd6788", "size": 1210152, "path": "media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4"}, "eval/num_success": 5.0, "episode": 15.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "60ba9e6f07a8e7ecafdb195bc5785362bfa055b68605ae8b6130e7223024372f", "size": 1207938, "path": "media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "7e5592f1dae1997ff968a60f1e3ba8250e00bfcb826bbdd9c7bf4480924e0788", "size": 1152050, "path": "media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "04fddf9f1781ca8c63e3cc9c73122b5e0a57d817663ee076dd71f11d71a3bc36", "size": 1259877, "path": "media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://6lcgxsu3x00aczdvqclhkq7ng43ahb0r6sgqi8zznfnly1s910edsj6rl6pzk08lbdeix1kfqki0gm3hoi5p146nnml9sbgavb4a7g4yzh9aox2pfsq54ktcg1onpc2e/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://abhicp1aggjs2mm1kssv5pegh9tvzynqcq1hfpboklpzvt0hhhmcml5v454fyt78w2ryyalbcmgqsps9m127f8amabwfj7dhdfq15o9nfh6plo6i62gji3hq7uhd549h:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 15, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9928352235762072, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7993280141947043, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.8249338715618673, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6e312070ee781a67b6ed562ed805cc283e04b6daa3f3dedf9db9ab8790e00e5b", "size": 420, "artifact_path": "wandb-client-artifact://ye77gqxk2nd54oka6pd29kktx020h8iciw1w6vdrm4l7um1qw3chqq2lmaa0t1d657usq721mly3z4i9wz25zui2o5hpl92rthmc90y141ihzqqxrgvcm3mm6n151brc/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ko2aiabdd4tpumnu309fvxu7xevirg15ojyciti5sete7ncv86vu4za4jq0lga7krjazuv4z3wjolq5jsb9zir8xfjzlo29qwn2laefm41vw5r6vumpglj7micvwjani:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 279}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9928352236747742, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7993280291557312, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8249338865280151, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug-internal.log b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d2116b3dd360d5bfef7cc146d54d7225f9d76ff5 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug-internal.log @@ -0,0 +1,810 @@ +2025-01-03 22:16:12,663 INFO StreamThr :1981451 [internal.py:wandb_internal():85] W&B internal server running at pid: 1981451, started at: 2025-01-03 22:16:12.662934 +2025-01-03 22:16:12,664 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:16:12,676 INFO WriterThread:1981451 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/run-ukc5qfsm.wandb +2025-01-03 22:16:12,677 DEBUG SenderThread:1981451 [sender.py:send():379] send: header +2025-01-03 22:16:12,684 DEBUG SenderThread:1981451 [sender.py:send():379] send: run +2025-01-03 22:16:12,928 INFO SenderThread:1981451 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files +2025-01-03 22:16:12,928 INFO SenderThread:1981451 [sender.py:_start_run_threads():1188] run started: ukc5qfsm with start time 1735960572.663657 +2025-01-03 22:16:12,943 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:16:12,944 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:16:12,988 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:16:12,993 DEBUG HandlerThread:1981451 [system_info.py:__init__():26] System info init +2025-01-03 22:16:12,993 DEBUG HandlerThread:1981451 [system_info.py:__init__():41] System info init done +2025-01-03 22:16:12,993 INFO HandlerThread:1981451 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:16:12,993 INFO SystemMonitor:1981451 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:16:12,993 INFO HandlerThread:1981451 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:16:12,994 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:16:12,994 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:16:12,995 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:16:12,995 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:16:12,996 INFO SystemMonitor:1981451 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:16:13,004 DEBUG HandlerThread:1981451 [system_info.py:probe():152] Probing system +2025-01-03 22:16:13,005 DEBUG HandlerThread:1981451 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_info.py:probe():200] Probing system done +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:16:13.004213', 'startedAt': '2025-01-04T03:16:12.653740', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.011375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.972, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 831.5409240722656}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:16:13,010 INFO HandlerThread:1981451 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:16:13,010 INFO HandlerThread:1981451 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:16:13,010 DEBUG HandlerThread:1981451 [system_info.py:_save_code():44] Saving code +2025-01-03 22:16:13,012 DEBUG HandlerThread:1981451 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:16:13,012 DEBUG HandlerThread:1981451 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:16:13,076 DEBUG HandlerThread:1981451 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:16:13,076 INFO HandlerThread:1981451 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:16:13,079 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:16:13,079 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:16:13,080 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:16:13,080 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:16:13,305 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/ks1ruya0-wandb-metadata.json +2025-01-03 22:16:13,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:16:13,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:16:13,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:13,319 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:13,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:13,321 INFO HandlerThread:1981451 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/tb" + save: true +} +control { + relay_id: "9d81edaa288d4dcfb7b8eac6a9e912ce" +} +_info { + stream_id: "ukc5qfsm" +} + +2025-01-03 22:16:13,321 DEBUG HandlerThread:1981451 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:16:13,354 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/fej5ou4n-code/FISH/eval_robot.py +2025-01-03 22:16:13,395 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/4rfg9bex-diff.patch +2025-01-03 22:16:13,459 DEBUG SenderThread:1981451 [sender.py:send():379] send: telemetry +2025-01-03 22:16:13,459 DEBUG SenderThread:1981451 [sender.py:send():379] send: tbrecord +2025-01-03 22:16:13,460 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:16:13,460 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 with policy live +2025-01-03 22:16:13,657 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/99uae6cr-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-metadata.json +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH/eval_robot.py +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/diff.patch +2025-01-03 22:16:13,930 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:16:13,931 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH +2025-01-03 22:16:13,931 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code +2025-01-03 22:16:14,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:15,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:15,929 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:16,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:17,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:18,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:18,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:19,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:20,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:21,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:22,109 DEBUG SenderThread:1981451 [sender.py:send():379] send: config +2025-01-03 22:16:22,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:23,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:23,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:23,931 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:24,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:25,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:26,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:27,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:27,933 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:28,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:28,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:28,361 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:28,442 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:29,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:30,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:31,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:31,934 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:32,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:33,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:34,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:34,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:35,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:36,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:37,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:38,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:39,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:40,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:40,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:41,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:42,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:43,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:43,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:43,365 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:44,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:45,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:45,415 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:45,937 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml +2025-01-03 22:16:46,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:47,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:48,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:49,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:49,939 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:50,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:50,725 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:51,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:51,939 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:52,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:53,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:53,940 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:16:54,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:55,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:56,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:16:56,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:57,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:58,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:16:58,318 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:16:58,362 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:16:59,324 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:00,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:01,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:01,400 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:02,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:03,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:04,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:05,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:06,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:07,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:07,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:08,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:09,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:10,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:11,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:11,945 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:12,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:12,856 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:12,996 DEBUG SystemMonitor:1981451 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:17:12,997 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:17:13,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:13,319 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:13,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:13,945 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:14,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:15,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:16,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:17,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:17,877 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:17,946 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:18,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:19,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:19,947 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:20,329 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:20,397 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:17:20,398 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:17:20,398 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 with policy now +2025-01-03 22:17:20,398 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:17:20,398 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:17:20,399 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:17:20,655 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/28g0bvwa-media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:17:20,947 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media +2025-01-03 22:17:20,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos +2025-01-03 22:17:21,274 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/6yk9q299-media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:17:21,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:21,948 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:22,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:23,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:23,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:24,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:25,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:25,949 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:26,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:27,329 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:27,950 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:28,318 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:28,319 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:28,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:28,423 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:28,950 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:17:29,200 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/2ev29hu3-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:17:29,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:29,950 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:30,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:31,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:31,951 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:32,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:33,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:33,952 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:34,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:34,330 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:35,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:35,953 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:36,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:37,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:38,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:39,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:40,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:40,331 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:41,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:42,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:43,005 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:17:43,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:43,320 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:43,361 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:44,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:45,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:45,956 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:46,273 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:46,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:47,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:48,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:49,332 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:50,333 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:51,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:51,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:52,349 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:53,349 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:54,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:55,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:56,351 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:56,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:17:57,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:57,960 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:17:58,319 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:17:58,320 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:17:58,361 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:17:59,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:00,350 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:01,354 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:01,521 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:01,777 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:18:01,777 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 with policy now +2025-01-03 22:18:01,777 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:18:01,778 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:18:01,778 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:18:01,779 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:18:01,961 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:18:01,961 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:01,961 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:18:01,962 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:18:02,044 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/h7qkclf6-media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:18:02,314 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/3uv2omx5-media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:18:02,354 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:03,359 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:03,962 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:04,353 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:05,356 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:06,355 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:06,541 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:07,360 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:07,963 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:08,357 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:09,360 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:09,964 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:10,360 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:11,359 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:12,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:12,369 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:12,998 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:18:13,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:13,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:13,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:14,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:15,367 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:16,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:17,367 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:17,450 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:18,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:19,366 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:20,367 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:21,369 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:22,370 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:23,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:23,368 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:24,370 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:25,369 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:26,373 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:27,372 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:28,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:28,323 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:28,372 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:28,468 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:29,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:30,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:31,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:32,374 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:32,765 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:18:32,765 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:18:32,765 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:18:32,972 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:18:32,972 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:18:33,216 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/1s94kxc5-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:18:33,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:33,972 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:34,326 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:34,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:35,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:36,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:37,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:37,974 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:38,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:39,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:39,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:39,974 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:40,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:41,377 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:42,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:42,998 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:18:43,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:43,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:43,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:44,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:44,438 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:45,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:46,378 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:47,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:47,977 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:48,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:49,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:49,978 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:18:50,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:50,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:51,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:52,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:53,379 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:54,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:55,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:56,327 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:18:56,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:57,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:58,320 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:18:58,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:18:58,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:18:59,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:00,380 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:01,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:01,444 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:02,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:03,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:04,381 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:05,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:05,770 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:19:05,770 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 with policy now +2025-01-03 22:19:05,773 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:19:05,774 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:05,774 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:05,774 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:19:05,983 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:19:06,023 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/8th1ww3u-media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:19:06,270 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/98tn6rf5-media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:19:06,385 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:06,783 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:07,384 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:07,984 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:08,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:09,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:10,393 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:11,386 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:11,985 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:12,267 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:12,385 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:12,999 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:19:13,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:13,322 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:13,387 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:13,986 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:14,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:15,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:15,986 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:16,393 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:16,600 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:16,600 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:16,601 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:16,987 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:16,987 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:17,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:17,987 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:18,021 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:18,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:19,399 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:20,388 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:21,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:21,988 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:22,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:23,253 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:23,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:24,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:25,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:26,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:27,389 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:28,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:28,321 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:28,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:28,410 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:29,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:30,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:31,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:32,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:33,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:34,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:34,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:35,390 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:36,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:37,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:38,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:39,391 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:40,328 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:40,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:41,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:42,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:42,999 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:19:43,321 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:43,322 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:43,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:44,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:45,376 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:45,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:46,392 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:47,406 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:47,997 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:48,069 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:19:48,069 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 with policy now +2025-01-03 22:19:48,072 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:19:48,072 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:48,072 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:48,073 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:48,352 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/5qtclipz-media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:19:48,393 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:48,997 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:48,998 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:19:48,998 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:19:49,287 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/fqbwdbex-media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:19:49,398 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:49,998 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:50,401 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:51,329 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:51,401 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:52,405 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:53,408 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:53,999 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:54,407 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:55,408 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:56,000 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:19:56,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:56,649 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:19:57,411 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:58,325 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:19:58,326 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:19:58,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:19:58,653 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:19:58,654 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:19:58,654 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:19:59,001 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:19:59,001 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:59,241 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/26qxe994-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:19:59,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:00,001 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:00,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:01,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:02,002 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:02,269 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:02,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:03,418 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:04,003 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:04,412 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:05,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:06,003 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:06,417 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:07,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:07,707 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:08,416 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:09,413 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:10,416 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:11,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:12,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:12,999 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:20:13,000 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:13,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:20:13,323 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:20:13,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:14,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:15,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:16,010 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:16,414 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:17,422 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:18,117 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:18,415 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:19,415 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:20,424 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:21,424 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:22,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:23,333 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:23,429 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:24,430 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:25,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:26,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:27,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:28,322 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:20:28,323 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:20:28,419 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:28,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:29,427 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:30,428 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:31,428 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:32,430 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:33,443 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:33,485 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:33,516 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:20:33,516 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 with policy now +2025-01-03 22:20:33,557 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:20:33,557 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:33,557 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:33,558 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:33,768 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/xqd8w5n6-media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:20:34,020 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval +2025-01-03 22:20:34,299 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/8m3q42gn-media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:20:34,441 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:35,443 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:36,021 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:36,440 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:37,449 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:38,444 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:39,340 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:39,442 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:40,022 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:40,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:41,512 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:41,712 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:41,712 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:41,713 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:41,788 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:20:41,789 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:20:41,984 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table +2025-01-03 22:20:42,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval +2025-01-03 22:20:42,204 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/60xfdf6p-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:20:42,317 INFO wandb-upload_2:1981451 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxci8vdsb +2025-01-03 22:20:42,349 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:20:42,359 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:20:42,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:42,716 INFO SenderThread:1981451 [sender.py:send_request_log_artifact():1525] logged artifact run-ukc5qfsm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjMwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTkzNg==', 'latestArtifact': None}} +2025-01-03 22:20:42,716 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:20:42,716 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-03 22:20:42,716 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:42,717 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,717 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,717 DEBUG SenderThread:1981451 [sender.py:send():379] send: telemetry +2025-01-03 22:20:42,717 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,721 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,721 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,721 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,721 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,722 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,722 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,722 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,722 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,723 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,723 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,723 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,723 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,723 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,724 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,724 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,724 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,724 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,725 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:42,725 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:42,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:20:42,980 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/u69p8n3n-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:20:43,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:43,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:20:43,023 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:20:43,024 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval +2025-01-03 22:20:43,107 INFO wandb-upload_2:1981451 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp420an72j +2025-01-03 22:20:43,226 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/vi5qq3r2-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:20:43,323 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:20:43,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:43,530 INFO SenderThread:1981451 [sender.py:send_request_log_artifact():1525] logged artifact run-ukc5qfsm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjMxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTk0MQ==', 'latestArtifact': None}} +2025-01-03 22:20:43,531 DEBUG SenderThread:1981451 [sender.py:send():379] send: files +2025-01-03 22:20:43,531 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json with policy now +2025-01-03 22:20:43,531 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:43,531 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:43,532 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:43,532 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:20:43,532 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:20:43,763 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/pj5thlno-media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:20:44,024 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:44,024 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:44,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:44,644 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:45,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:46,508 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:47,513 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:48,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:49,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:50,335 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:50,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:51,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:52,509 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:20:52,613 DEBUG SenderThread:1981451 [sender.py:send():379] send: exit +2025-01-03 22:20:52,613 INFO SenderThread:1981451 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:20:52,613 INFO SenderThread:1981451 [sender.py:send_exit():588] handling runtime: 279 +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:send_exit():594] send defer +2025-01-03 22:20:52,614 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,614 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:20:52,614 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:20:52,614 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:20:52,614 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,614 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:20:52,615 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,615 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:20:52,615 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:20:52,615 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,615 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:20:52,615 INFO HandlerThread:1981451 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:20:52,615 DEBUG SystemMonitor:1981451 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:20:52,615 DEBUG SystemMonitor:1981451 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:20:52,616 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:20:52,616 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:20:52,686 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:20:52,686 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:20:52,687 INFO HandlerThread:1981451 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:20:52,687 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,687 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:20:52,687 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:20:52,687 DEBUG SenderThread:1981451 [sender.py:send():379] send: stats +2025-01-03 22:20:52,687 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,687 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:20:52,688 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:52,688 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:20:52,688 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:20:52,688 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:52,688 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:20:53,027 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:54,028 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:20:58,718 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:20:58,718 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:58,718 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,719 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:20:58,719 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send():379] send: history +2025-01-03 22:20:58,719 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:20:58,720 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:58,720 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:20:58,720 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:58,720 DEBUG SenderThread:1981451 [sender.py:send():379] send: summary +2025-01-03 22:20:58,724 INFO SenderThread:1981451 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:20:58,724 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,724 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:20:58,724 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:20:58,725 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:58,725 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:20:58,725 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,725 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:20:58,728 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:20:58,826 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:20:58,826 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:58,826 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:20:58,826 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:58,826 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:20:59,030 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:20:59,030 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml +2025-01-03 22:20:59,614 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:20:59,761 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:20:59,761 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:20:59,761 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:59,761 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:20:59,762 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:20:59,762 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:20:59,762 INFO SenderThread:1981451 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:20:59,764 INFO SenderThread:1981451 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:20:59,766 INFO SenderThread:1981451 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:20:59,779 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:20:59,780 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:20:59,780 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:20:59,780 DEBUG SenderThread:1981451 [sender.py:send():379] send: artifact +2025-01-03 22:21:00,030 INFO Thread-12 :1981451 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:21:00,125 INFO wandb-upload_2:1981451 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyehzxc9p +2025-01-03 22:21:00,125 INFO wandb-upload_1:1981451 [upload_job.py:push():85] Skipped uploading /tmp/tmpnmd_mrlg/wandb-job.json +2025-01-03 22:21:00,342 INFO wandb-upload_0:1981451 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwj6kj7gm +2025-01-03 22:21:00,615 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:21:00,901 INFO SenderThread:1981451 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'versionIndex': 411}}} +2025-01-03 22:21:00,901 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:00,901 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:21:00,902 INFO SenderThread:1981451 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml config.yaml +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json wandb-summary.json +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt requirements.txt +2025-01-03 22:21:01,031 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:21:01,032 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:21:01,032 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log output.log +2025-01-03 22:21:01,032 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/diff.patch diff.patch +2025-01-03 22:21:01,036 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 media/videos/eval/1_eval_1_e3a6cf8eaa58e555adc3.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 media/videos/eval/0_eval_0_00fe423cbd21a3f93a8f.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 media/videos/eval/4_eval_4_04fddf9f1781ca8c63e3.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 media/videos/eval/2_eval_2_60ba9e6f07a8e7ecafdb.mp4 +2025-01-03 22:21:01,038 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 media/videos/eval/3_eval_3_7e5592f1dae1997ff968.mp4 +2025-01-03 22:21:01,040 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json media/table/eval/total_success_rate_ci_6_6e312070ee781a67b6ed.table.json +2025-01-03 22:21:01,040 INFO SenderThread:1981451 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 22:21:01,040 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:21:01,040 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:21:01,040 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:21:01,040 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:21:01,041 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:01,041 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:21:01,041 INFO SenderThread:1981451 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:21:01,216 INFO wandb-upload_0:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/requirements.txt +2025-01-03 22:21:01,219 INFO wandb-upload_2:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/config.yaml +2025-01-03 22:21:01,259 INFO wandb-upload_1:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/wandb-summary.json +2025-01-03 22:21:01,279 INFO wandb-upload_4:1981451 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/files/output.log +2025-01-03 22:21:01,326 INFO wandb-upload_3:1981451 [upload_job.py:push():130] Uploaded file /tmp/tmpyu9ny5ttwandb/fj4w5byq-events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0 +2025-01-03 22:21:01,527 INFO Thread-11 (_thread_body):1981451 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:21:01,527 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:21:01,527 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:21:01,528 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:01,528 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:21:01,528 INFO SenderThread:1981451 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:21:01,531 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:21:01,531 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:21:01,531 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:21:01,531 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:21:01,531 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:21:01,531 INFO SenderThread:1981451 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:21:01,615 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:21:06,617 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:11,618 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:16,619 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:21,620 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:26,622 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:31,623 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:36,624 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:41,625 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:46,626 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:51,628 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:21:56,629 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:22:01,630 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:22:01,657 WARNING FileStreamThread:1981451 [file_stream.py:request_with_retry():674] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/serialexperimentsleon/extrinsic_contact_downstream/ukc5qfsm/file_stream. func: >, args: ('https://api.wandb.ai/files/serialexperimentsleon/extrinsic_contact_downstream/ukc5qfsm/file_stream',), kwargs: {'json': {'complete': False, 'failed': False, 'dropped': 0, 'uploaded': ['wandb-summary.json', 'requirements.frozen.txt', 'diff.patch', 'wandb-job.json', 'config.yaml', 'output.log', 'requirements.txt', 'events.out.tfevents.1735960573.leonmkim-ROG-Strix-G15CS-G15CS.1981219.0']}} +2025-01-03 22:22:03,821 INFO SenderThread:1981451 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:22:03,821 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:22:03,821 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:22:03,821 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:22:03,821 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:22:03,821 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:22:03,822 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:22:03,822 INFO SenderThread:1981451 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:22:03,822 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send():379] send: final +2025-01-03 22:22:03,822 INFO HandlerThread:1981451 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send():379] send: footer +2025-01-03 22:22:03,822 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: defer +2025-01-03 22:22:03,822 INFO SenderThread:1981451 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:22:03,823 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:22:03,823 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:22:03,823 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:22:03,823 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:22:03,824 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:22:03,825 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:22:03,826 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:22:03,826 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:22:03,906 DEBUG SenderThread:1981451 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:22:03,906 INFO MainThread:1981451 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:22:03,907 INFO MainThread:1981451 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:22:03,907 INFO MainThread:1981451 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:22:03,907 DEBUG HandlerThread:1981451 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:22:03,907 INFO HandlerThread:1981451 [handler.py:finish():882] shutting down handler +2025-01-03 22:22:04,824 INFO WriterThread:1981451 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/run-ukc5qfsm.wandb +2025-01-03 22:22:04,906 INFO SenderThread:1981451 [sender.py:finish():1615] shutting down sender +2025-01-03 22:22:04,906 INFO SenderThread:1981451 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:22:04,906 INFO SenderThread:1981451 [file_pusher.py:join():175] waiting for file pusher diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug.log b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1317f07e40f7c906f31d10f10172420b3e1b3d38 --- /dev/null +++ b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Configure stats pid to 1981219 +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/settings +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug.log +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/logs/debug-internal.log +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm'} +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():619] starting backend +2025-01-03 22:16:12,655 INFO MainThread:1981219 [wandb_init.py:init():623] setting up manager +2025-01-03 22:16:12,662 INFO MainThread:1981219 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:16:12,663 INFO MainThread:1981219 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:16:12,676 INFO MainThread:1981219 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:16:12,682 INFO MainThread:1981219 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:16:12,943 INFO MainThread:1981219 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:16:12,983 INFO MainThread:1981219 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:16:12,984 INFO MainThread:1981219 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:16:13,318 INFO MainThread:1981219 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:16:13,318 INFO MainThread:1981219 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:16:13,319 INFO MainThread:1981219 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:16:13,319 INFO MainThread:1981219 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:16:13,320 INFO MainThread:1981219 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:16:13,321 INFO MainThread:1981219 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ukc5qfsm/tb, True +2025-01-03 22:16:22,108 INFO MainThread:1981219 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:22:04,908 WARNING MsgRouterThr:1981219 [router.py:message_loop():77] message_loop has been closed diff --git a/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/run-ukc5qfsm.wandb b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/run-ukc5qfsm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ef3acaf7f315a6b2b61e6ad151a1f240a154bc88 Binary files /dev/null and b/ukc5qfsm/wandb/run-20250103_221612-ukc5qfsm/run-ukc5qfsm.wandb differ