Add files using upload-large-folder tool
Browse files- Abs_6D + DeltaEE/Baseline/checkpoints/060000/source_config.yaml +63 -0
- Abs_6D + DeltaEE/Baseline/resume_gpu1_20260721.log +0 -0
- Abs_6D + DeltaEE/Baseline/wandb/debug.log +51 -31
- Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/config.yaml +333 -0
- Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/output.log +174 -0
- Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/requirements.txt +117 -0
- Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/wandb-summary.json +1 -0
- Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug-core.log +16 -0
- Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug.log +65 -0
Abs_6D + DeltaEE/Baseline/checkpoints/060000/source_config.yaml
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# INSIGHT GR00T N1.7 training config
|
| 2 |
+
# INSIGHT wrapper-only fields are stripped before LeRobot config parsing.
|
| 3 |
+
insight:
|
| 4 |
+
# Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta
|
| 5 |
+
action_space: ee_abs_rot6d+ee_delta
|
| 6 |
+
# Options: coarse | detailed
|
| 7 |
+
prompt_set: detailed
|
| 8 |
+
save_steps: ["60K"]
|
| 9 |
+
|
| 10 |
+
dataset:
|
| 11 |
+
repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
|
| 12 |
+
root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2
|
| 13 |
+
|
| 14 |
+
policy:
|
| 15 |
+
type: groot
|
| 16 |
+
device: cuda
|
| 17 |
+
chunk_size: 16
|
| 18 |
+
n_action_steps: 16
|
| 19 |
+
push_to_hub: false
|
| 20 |
+
# Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen.
|
| 21 |
+
# VLM
|
| 22 |
+
tune_llm: false
|
| 23 |
+
tune_visual: false
|
| 24 |
+
tune_top_llm_layers: 0
|
| 25 |
+
# Action Expert
|
| 26 |
+
tune_projector: true
|
| 27 |
+
tune_diffusion_model: true
|
| 28 |
+
tune_vlln: true
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
seed: 42
|
| 32 |
+
batch_size: 128
|
| 33 |
+
steps: 60000
|
| 34 |
+
log_freq: 200
|
| 35 |
+
output_dir: "${HOME}/groot_insight/train_ckpt/Abs_6D + DeltaEE/Baseline"
|
| 36 |
+
job_name: INSIGHT_RD_baseline
|
| 37 |
+
|
| 38 |
+
wandb:
|
| 39 |
+
enable: true
|
| 40 |
+
disable_artifact: true
|
| 41 |
+
|
| 42 |
+
# Omitted because these match LeRobot/GR00T defaults or are unused in this setup:
|
| 43 |
+
#
|
| 44 |
+
# dataset:
|
| 45 |
+
# eval_split: 0.0
|
| 46 |
+
# image_transforms:
|
| 47 |
+
# enable: false # default; set true only for train-image augmentation.
|
| 48 |
+
#
|
| 49 |
+
# policy:
|
| 50 |
+
# base_model_path: nvidia/GR00T-N1.7-3B
|
| 51 |
+
# embodiment_tag: new_embodiment
|
| 52 |
+
# use_relative_actions: false
|
| 53 |
+
# use_bf16: true
|
| 54 |
+
# repo_id: Whalswp/INSIGHT # unused while push_to_hub is false
|
| 55 |
+
#
|
| 56 |
+
# save_checkpoint: true
|
| 57 |
+
# use_policy_training_preset: true
|
| 58 |
+
# eval_steps: 0
|
| 59 |
+
# env_eval_freq: 0
|
| 60 |
+
#
|
| 61 |
+
# insight.prompt_set options:
|
| 62 |
+
# coarse: High-level task text that requires following the visual guide.
|
| 63 |
+
# detailed: Direction- and interaction-specific text, including rotation and push/pull details.
|
Abs_6D + DeltaEE/Baseline/resume_gpu1_20260721.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
Abs_6D + DeltaEE/Baseline/wandb/debug.log
CHANGED
|
@@ -1,45 +1,65 @@
|
|
| 1 |
-
2026-07-
|
| 2 |
-
2026-07-
|
| 3 |
-
2026-07-
|
| 4 |
-
2026-07-
|
| 5 |
-
2026-07-
|
| 6 |
-
2026-07-
|
| 7 |
-
2026-07-
|
| 8 |
-
config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/
|
| 9 |
-
2026-07-
|
| 10 |
-
2026-07-
|
| 11 |
-
2026-07-
|
| 12 |
-
2026-07-
|
| 13 |
-
2026-07-
|
| 14 |
-
2026-07-
|
| 15 |
-
2026-07-
|
| 16 |
-
2026-07-
|
| 17 |
-
2026-07-
|
| 18 |
-
2026-07-
|
| 19 |
-
2026-07-
|
| 20 |
-
2026-07-
|
| 21 |
-
2026-07-
|
| 22 |
-
2026-07-
|
|
|
|
| 23 |
Traceback (most recent call last):
|
| 24 |
-
File "/home/
|
| 25 |
await fn()
|
| 26 |
-
File "/home/
|
| 27 |
await asyncio_compat.race(
|
| 28 |
-
File "/home/
|
| 29 |
async with open_task_group(race=True) as tg:
|
| 30 |
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 31 |
-
File "/home/
|
| 32 |
await anext(self.gen)
|
| 33 |
-
File "/home/
|
| 34 |
await task_group._wait_all(race=race, timeout=exit_timeout)
|
| 35 |
-
File "/home/
|
| 36 |
raise exc
|
| 37 |
-
File "/home/
|
| 38 |
result = await handle.wait_async(timeout=None)
|
| 39 |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 40 |
-
File "/home/
|
| 41 |
response = await self._handle.wait_async(timeout=timeout)
|
| 42 |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 43 |
-
File "/home/
|
| 44 |
raise HandleAbandonedError()
|
| 45 |
wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
|
| 2 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_setup.py:_flush():81] Configure stats pid to 1492997
|
| 3 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug.log
|
| 5 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug-internal.log
|
| 6 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:init():766] calling init triggers
|
| 7 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/seonho/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [16]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'h160hqm2', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000', '_wandb': {}}
|
| 9 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:init():814] starting backend
|
| 10 |
+
2026-07-21 13:48:50,633 INFO MainThread:1492997 [wandb_init.py:init():829] sending inform_init request
|
| 11 |
+
2026-07-21 13:48:51,097 INFO MainThread:1492997 [wandb_init.py:init():834] backend started and connected
|
| 12 |
+
2026-07-21 13:48:51,099 INFO MainThread:1492997 [wandb_init.py:init():904] updated telemetry
|
| 13 |
+
2026-07-21 13:48:51,105 INFO MainThread:1492997 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-21 13:48:51,906 INFO MainThread:1492997 [wandb_init.py:init():967] run resumed
|
| 15 |
+
2026-07-21 13:48:51,912 INFO MainThread:1492997 [wandb_init.py:init():972] starting run threads in backend
|
| 16 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_console_start():2523] atexit reg
|
| 17 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 18 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 19 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 20 |
+
2026-07-21 13:48:52,050 INFO MainThread:1492997 [wandb_init.py:init():1010] run started, returning control to user process
|
| 21 |
+
2026-07-22 20:07:27,908 INFO wandb-AsyncioManager-main:1492997 [service_client.py:_forward_responses():122] Reached EOF.
|
| 22 |
+
2026-07-22 20:07:27,909 INFO wandb-AsyncioManager-main:1492997 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
|
| 23 |
+
2026-07-22 20:07:27,910 ERROR wandb-AsyncioManager-main:1492997 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
|
| 24 |
Traceback (most recent call last):
|
| 25 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
|
| 26 |
await fn()
|
| 27 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
|
| 28 |
await asyncio_compat.race(
|
| 29 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
|
| 30 |
async with open_task_group(race=True) as tg:
|
| 31 |
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 32 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
|
| 33 |
await anext(self.gen)
|
| 34 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
|
| 35 |
await task_group._wait_all(race=race, timeout=exit_timeout)
|
| 36 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
|
| 37 |
raise exc
|
| 38 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
|
| 39 |
result = await handle.wait_async(timeout=None)
|
| 40 |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 41 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
|
| 42 |
response = await self._handle.wait_async(timeout=timeout)
|
| 43 |
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 44 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
|
| 45 |
raise HandleAbandonedError()
|
| 46 |
wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
|
| 47 |
+
2026-07-22 20:07:33,323 ERROR wandb-AsyncioManager-main:1492997 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
|
| 48 |
+
Traceback (most recent call last):
|
| 49 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
|
| 50 |
+
await fn()
|
| 51 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
|
| 52 |
+
should_stop = await self._check_should_stop()
|
| 53 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 54 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
|
| 55 |
+
handle = await self._interface.deliver_async(request)
|
| 56 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 57 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
|
| 58 |
+
handle = await self._client.deliver(request)
|
| 59 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 60 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
|
| 61 |
+
handle = self._mailbox.require_response(request)
|
| 62 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 63 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
|
| 64 |
+
raise MailboxClosedError()
|
| 65 |
+
wandb.sdk.mailbox.mailbox.MailboxClosedError
|
Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/config.yaml
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.27.2
|
| 4 |
+
e:
|
| 5 |
+
iqbqf28c5ojr7e18m3sfpy2lxrf27dgv:
|
| 6 |
+
args:
|
| 7 |
+
- --config_path=/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000/pretrained_model/train_config.json
|
| 8 |
+
- --resume=true
|
| 9 |
+
- --dataset.root=/home/seonho/INSIGHTfixposV4_filtered_multispace_v2
|
| 10 |
+
- --output_dir=/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline
|
| 11 |
+
codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
|
| 12 |
+
codePathLocal: my_scripts/lerobot_train_yaml.py
|
| 13 |
+
cpu_count: 32
|
| 14 |
+
cpu_count_logical: 64
|
| 15 |
+
cudaVersion: "13.2"
|
| 16 |
+
disk:
|
| 17 |
+
/:
|
| 18 |
+
total: "3778130485248"
|
| 19 |
+
used: "3198141472768"
|
| 20 |
+
executable: /home/seonho/miniconda3/envs/lerobot060_groot/bin/python
|
| 21 |
+
git:
|
| 22 |
+
commit: a9df08f7da2d8eb1fa7f28708b4eb6f6dc81b1f9
|
| 23 |
+
remote: git@github.com:paragon7060/clvla.git
|
| 24 |
+
gpu: NVIDIA A100 80GB PCIe
|
| 25 |
+
gpu_count: 4
|
| 26 |
+
gpu_nvidia:
|
| 27 |
+
- architecture: Ampere
|
| 28 |
+
cudaCores: 6912
|
| 29 |
+
memoryTotal: "85899345920"
|
| 30 |
+
name: NVIDIA A100 80GB PCIe
|
| 31 |
+
uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9
|
| 32 |
+
- architecture: Ampere
|
| 33 |
+
cudaCores: 6912
|
| 34 |
+
memoryTotal: "85899345920"
|
| 35 |
+
name: NVIDIA A100 80GB PCIe
|
| 36 |
+
uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa
|
| 37 |
+
- architecture: Ampere
|
| 38 |
+
cudaCores: 6912
|
| 39 |
+
memoryTotal: "85899345920"
|
| 40 |
+
name: NVIDIA A100 80GB PCIe
|
| 41 |
+
uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac
|
| 42 |
+
- architecture: Ampere
|
| 43 |
+
cudaCores: 6912
|
| 44 |
+
memoryTotal: "85899345920"
|
| 45 |
+
name: NVIDIA A100 80GB PCIe
|
| 46 |
+
uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d
|
| 47 |
+
host: worker1
|
| 48 |
+
memory:
|
| 49 |
+
total: "1081620799488"
|
| 50 |
+
os: Linux-6.8.0-110-generic-x86_64-with-glibc2.35
|
| 51 |
+
program: /home/seonho/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py
|
| 52 |
+
python: CPython 3.12.13
|
| 53 |
+
root: /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline
|
| 54 |
+
startedAt: "2026-07-21T04:48:50.415152Z"
|
| 55 |
+
writerId: iqbqf28c5ojr7e18m3sfpy2lxrf27dgv
|
| 56 |
+
m: []
|
| 57 |
+
python_version: 3.12.13
|
| 58 |
+
t:
|
| 59 |
+
"1":
|
| 60 |
+
- 1
|
| 61 |
+
- 11
|
| 62 |
+
- 41
|
| 63 |
+
- 49
|
| 64 |
+
- 51
|
| 65 |
+
- 71
|
| 66 |
+
- 83
|
| 67 |
+
- 98
|
| 68 |
+
"2":
|
| 69 |
+
- 1
|
| 70 |
+
- 11
|
| 71 |
+
- 41
|
| 72 |
+
- 49
|
| 73 |
+
- 51
|
| 74 |
+
- 71
|
| 75 |
+
- 83
|
| 76 |
+
- 98
|
| 77 |
+
"3":
|
| 78 |
+
- 5
|
| 79 |
+
- 13
|
| 80 |
+
- 14
|
| 81 |
+
- 15
|
| 82 |
+
- 16
|
| 83 |
+
- 61
|
| 84 |
+
- 62
|
| 85 |
+
"4": 3.12.13
|
| 86 |
+
"5": 0.27.2
|
| 87 |
+
"6": 5.5.4
|
| 88 |
+
"10":
|
| 89 |
+
- 21
|
| 90 |
+
"12": 0.27.2
|
| 91 |
+
"13": linux-x86_64
|
| 92 |
+
batch_size:
|
| 93 |
+
value: 128
|
| 94 |
+
checkpoint_path:
|
| 95 |
+
value: /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000
|
| 96 |
+
cudnn_deterministic:
|
| 97 |
+
value: false
|
| 98 |
+
dataset:
|
| 99 |
+
value:
|
| 100 |
+
depth_output_unit: mm
|
| 101 |
+
episodes: null
|
| 102 |
+
eval_split: 0
|
| 103 |
+
image_transforms:
|
| 104 |
+
enable: false
|
| 105 |
+
max_num_transforms: 3
|
| 106 |
+
random_order: false
|
| 107 |
+
tfs:
|
| 108 |
+
affine:
|
| 109 |
+
kwargs:
|
| 110 |
+
degrees:
|
| 111 |
+
- -5
|
| 112 |
+
- 5
|
| 113 |
+
translate:
|
| 114 |
+
- 0.05
|
| 115 |
+
- 0.05
|
| 116 |
+
type: RandomAffine
|
| 117 |
+
weight: 1
|
| 118 |
+
brightness:
|
| 119 |
+
kwargs:
|
| 120 |
+
brightness:
|
| 121 |
+
- 0.8
|
| 122 |
+
- 1.2
|
| 123 |
+
type: ColorJitter
|
| 124 |
+
weight: 1
|
| 125 |
+
contrast:
|
| 126 |
+
kwargs:
|
| 127 |
+
contrast:
|
| 128 |
+
- 0.8
|
| 129 |
+
- 1.2
|
| 130 |
+
type: ColorJitter
|
| 131 |
+
weight: 1
|
| 132 |
+
hue:
|
| 133 |
+
kwargs:
|
| 134 |
+
hue:
|
| 135 |
+
- -0.05
|
| 136 |
+
- 0.05
|
| 137 |
+
type: ColorJitter
|
| 138 |
+
weight: 1
|
| 139 |
+
saturation:
|
| 140 |
+
kwargs:
|
| 141 |
+
saturation:
|
| 142 |
+
- 0.5
|
| 143 |
+
- 1.5
|
| 144 |
+
type: ColorJitter
|
| 145 |
+
weight: 1
|
| 146 |
+
sharpness:
|
| 147 |
+
kwargs:
|
| 148 |
+
sharpness:
|
| 149 |
+
- 0.5
|
| 150 |
+
- 1.5
|
| 151 |
+
type: SharpnessJitter
|
| 152 |
+
weight: 1
|
| 153 |
+
repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2
|
| 154 |
+
return_uint8: false
|
| 155 |
+
revision: null
|
| 156 |
+
root: /home/seonho/INSIGHTfixposV4_filtered_multispace_v2
|
| 157 |
+
streaming: false
|
| 158 |
+
use_imagenet_stats: true
|
| 159 |
+
video_backend: torchcodec
|
| 160 |
+
env:
|
| 161 |
+
value: null
|
| 162 |
+
env_eval_freq:
|
| 163 |
+
value: 20000
|
| 164 |
+
eval:
|
| 165 |
+
value:
|
| 166 |
+
batch_size: 50
|
| 167 |
+
n_episodes: 50
|
| 168 |
+
recording: false
|
| 169 |
+
recording_private: false
|
| 170 |
+
recording_repo_id: null
|
| 171 |
+
use_async_envs: true
|
| 172 |
+
eval_steps:
|
| 173 |
+
value: 0
|
| 174 |
+
job:
|
| 175 |
+
value:
|
| 176 |
+
detach: false
|
| 177 |
+
image: huggingface/lerobot-gpu:latest
|
| 178 |
+
tags: []
|
| 179 |
+
target: null
|
| 180 |
+
timeout: 2d
|
| 181 |
+
job_name:
|
| 182 |
+
value: INSIGHT_RD_baseline
|
| 183 |
+
log_freq:
|
| 184 |
+
value: 200
|
| 185 |
+
max_eval_samples:
|
| 186 |
+
value: 0
|
| 187 |
+
num_workers:
|
| 188 |
+
value: 4
|
| 189 |
+
optimizer:
|
| 190 |
+
value:
|
| 191 |
+
betas:
|
| 192 |
+
- 0.9
|
| 193 |
+
- 0.999
|
| 194 |
+
eps: 1e-08
|
| 195 |
+
grad_clip_norm: 1
|
| 196 |
+
lr: 0.0001
|
| 197 |
+
type: adamw
|
| 198 |
+
weight_decay: 1e-05
|
| 199 |
+
output_dir:
|
| 200 |
+
value: /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline
|
| 201 |
+
peft:
|
| 202 |
+
value: null
|
| 203 |
+
persistent_workers:
|
| 204 |
+
value: true
|
| 205 |
+
policy:
|
| 206 |
+
value:
|
| 207 |
+
action_decode_transform: null
|
| 208 |
+
balance_dataset_weights: true
|
| 209 |
+
balance_trajectory_weights: true
|
| 210 |
+
base_model_path: nvidia/GR00T-N1.7-3B
|
| 211 |
+
batch_size: 32
|
| 212 |
+
chunk_size: 16
|
| 213 |
+
dataloader_num_workers: 8
|
| 214 |
+
dataset_paths: null
|
| 215 |
+
device: cuda
|
| 216 |
+
embodiment_tag: new_embodiment
|
| 217 |
+
image_size:
|
| 218 |
+
- 256
|
| 219 |
+
- 256
|
| 220 |
+
input_features:
|
| 221 |
+
observation.images.guide:
|
| 222 |
+
shape:
|
| 223 |
+
- 3
|
| 224 |
+
- 224
|
| 225 |
+
- 224
|
| 226 |
+
type: VISUAL
|
| 227 |
+
observation.images.right_shoulder:
|
| 228 |
+
shape:
|
| 229 |
+
- 3
|
| 230 |
+
- 224
|
| 231 |
+
- 224
|
| 232 |
+
type: VISUAL
|
| 233 |
+
observation.images.wrist:
|
| 234 |
+
shape:
|
| 235 |
+
- 3
|
| 236 |
+
- 224
|
| 237 |
+
- 224
|
| 238 |
+
type: VISUAL
|
| 239 |
+
observation.state:
|
| 240 |
+
shape:
|
| 241 |
+
- 16
|
| 242 |
+
type: STATE
|
| 243 |
+
license: null
|
| 244 |
+
lora_alpha: 16
|
| 245 |
+
lora_dropout: 0.1
|
| 246 |
+
lora_full_model: false
|
| 247 |
+
lora_rank: 0
|
| 248 |
+
max_action_dim: 132
|
| 249 |
+
max_state_dim: 132
|
| 250 |
+
max_steps: 10000
|
| 251 |
+
model_params_fp32: true
|
| 252 |
+
n_action_steps: 16
|
| 253 |
+
n_obs_steps: 1
|
| 254 |
+
normalization_mapping:
|
| 255 |
+
ACTION: IDENTITY
|
| 256 |
+
STATE: IDENTITY
|
| 257 |
+
VISUAL: IDENTITY
|
| 258 |
+
num_inference_timesteps: null
|
| 259 |
+
optimizer_betas:
|
| 260 |
+
- 0.9
|
| 261 |
+
- 0.999
|
| 262 |
+
optimizer_eps: 1e-08
|
| 263 |
+
optimizer_lr: 0.0001
|
| 264 |
+
optimizer_weight_decay: 1e-05
|
| 265 |
+
output_dir: ./tmp/gr00t
|
| 266 |
+
output_features:
|
| 267 |
+
action:
|
| 268 |
+
shape:
|
| 269 |
+
- 16
|
| 270 |
+
type: ACTION
|
| 271 |
+
pretrained_path: /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000/pretrained_model
|
| 272 |
+
pretrained_revision: null
|
| 273 |
+
private: null
|
| 274 |
+
push_to_hub: false
|
| 275 |
+
relative_exclude_joints: []
|
| 276 |
+
repo_id: null
|
| 277 |
+
report_to: wandb
|
| 278 |
+
resume: false
|
| 279 |
+
rtc_ramp_rate: null
|
| 280 |
+
save_steps: 1000
|
| 281 |
+
tags: null
|
| 282 |
+
tokenizer_assets_repo: null
|
| 283 |
+
tune_diffusion_model: true
|
| 284 |
+
tune_llm: false
|
| 285 |
+
tune_projector: true
|
| 286 |
+
tune_top_llm_layers: 0
|
| 287 |
+
tune_visual: false
|
| 288 |
+
tune_vlln: true
|
| 289 |
+
type: groot
|
| 290 |
+
use_amp: false
|
| 291 |
+
use_bf16: true
|
| 292 |
+
use_flash_attention: false
|
| 293 |
+
use_peft: false
|
| 294 |
+
use_relative_actions: false
|
| 295 |
+
video_backend: decord
|
| 296 |
+
warmup_ratio: 0.05
|
| 297 |
+
prefetch_factor:
|
| 298 |
+
value: 4
|
| 299 |
+
resume:
|
| 300 |
+
value: true
|
| 301 |
+
reward_model:
|
| 302 |
+
value: null
|
| 303 |
+
sample_weighting:
|
| 304 |
+
value: null
|
| 305 |
+
save_checkpoint:
|
| 306 |
+
value: true
|
| 307 |
+
save_checkpoint_to_hub:
|
| 308 |
+
value: false
|
| 309 |
+
save_freq:
|
| 310 |
+
value: 20000
|
| 311 |
+
scheduler:
|
| 312 |
+
value:
|
| 313 |
+
name: cosine
|
| 314 |
+
num_warmup_steps: 500
|
| 315 |
+
type: diffuser
|
| 316 |
+
seed:
|
| 317 |
+
value: 42
|
| 318 |
+
steps:
|
| 319 |
+
value: 60000
|
| 320 |
+
tolerance_s:
|
| 321 |
+
value: 0.0001
|
| 322 |
+
use_policy_training_preset:
|
| 323 |
+
value: true
|
| 324 |
+
wandb:
|
| 325 |
+
value:
|
| 326 |
+
add_tags: true
|
| 327 |
+
disable_artifact: true
|
| 328 |
+
enable: true
|
| 329 |
+
entity: null
|
| 330 |
+
mode: null
|
| 331 |
+
notes: null
|
| 332 |
+
project: lerobot
|
| 333 |
+
run_id: h160hqm2
|
Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/output.log
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
INFO 2026-07-21 13:48:52 db_utils.py:121 Logs will be synced with wandb.
|
| 2 |
+
INFO 2026-07-21 13:48:52 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/h160hqm2
|
| 3 |
+
INFO 2026-07-21 13:48:52 ot_train.py:298 Creating dataset
|
| 4 |
+
INFO 2026-07-21 13:48:54 ot_train.py:332 Creating policy
|
| 5 |
+
INFO 2026-07-21 13:48:54 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000/pretrained_model
|
| 6 |
+
INFO 2026-07-21 13:48:54 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict...
|
| 7 |
+
Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 4156.43it/s]
|
| 8 |
+
`torch_dtype` is deprecated! Use `dtype` instead!
|
| 9 |
+
Loading weights: 100%|██████████| 1031/1031 [00:07<00:00, 146.43it/s]
|
| 10 |
+
Loading weights from local directory
|
| 11 |
+
INFO 2026-07-21 13:49:22 ot_train.py:405 Creating optimizer and scheduler
|
| 12 |
+
INFO 2026-07-21 13:49:25 ot_train.py:437 Output dir: /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline
|
| 13 |
+
INFO 2026-07-21 13:49:25 ot_train.py:444 cfg.steps=60000 (60K)
|
| 14 |
+
INFO 2026-07-21 13:49:25 ot_train.py:445 dataset.num_frames=218367 (218K)
|
| 15 |
+
INFO 2026-07-21 13:49:25 ot_train.py:446 dataset.num_episodes=4930
|
| 16 |
+
INFO 2026-07-21 13:49:25 ot_train.py:449 Effective batch size: 128 x 1 = 128
|
| 17 |
+
INFO 2026-07-21 13:49:25 ot_train.py:450 num_learnable_params=1620515968 (2B)
|
| 18 |
+
INFO 2026-07-21 13:49:25 ot_train.py:451 num_total_params=3144016000 (3B)
|
| 19 |
+
INFO 2026-07-21 13:49:25 ot_train.py:492 Resuming data order at epoch 26, sample 82688
|
| 20 |
+
Training: 0%| | 0/30000 [00:00<?, ?step/s]INFO 2026-07-21 13:49:26 ot_train.py:602 Start offline training on a fixed dataset, with effective batch size: 128
|
| 21 |
+
Training: 1%| | 200/30000 [12:35<30:55:37, 3.74s/step]INFO 2026-07-21 14:02:02 ot_train.py:646 step:30K smpl:4M ep:87K epch:17.70 loss:0.018 grdn:0.147 lr:5.0e-05 updt_s:2.489 data_s:1.287 smp/s:34 mem_gb:52.80
|
| 22 |
+
Training: 1%|▏ | 400/30000 [25:36<33:35:44, 4.09s/step]INFO 2026-07-21 14:15:02 ot_train.py:646 step:30K smpl:4M ep:88K epch:17.82 loss:0.018 grdn:0.146 lr:5.0e-05 updt_s:2.530 data_s:1.371 smp/s:33 mem_gb:52.80
|
| 23 |
+
Training: 2%|▏ | 600/30000 [39:43<36:41:55, 4.49s/step]INFO 2026-07-21 14:29:09 ot_train.py:646 step:31K smpl:4M ep:88K epch:17.94 loss:0.018 grdn:0.151 lr:4.9e-05 updt_s:2.667 data_s:1.564 smp/s:30 mem_gb:52.73
|
| 24 |
+
Training: 3%|▎ | 800/30000 [54:44<32:58:10, 4.06s/step]INFO 2026-07-21 14:44:10 ot_train.py:646 step:31K smpl:4M ep:89K epch:18.05 loss:0.017 grdn:0.146 lr:4.9e-05 updt_s:2.927 data_s:1.575 smp/s:28 mem_gb:52.82
|
| 25 |
+
Training: 3%|▎ | 1000/30000 [1:08:58<32:36:47, 4.05s/step]INFO 2026-07-21 14:58:25 ot_train.py:646 step:31K smpl:4M ep:90K epch:18.17 loss:0.017 grdn:0.145 lr:4.8e-05 updt_s:2.677 data_s:1.594 smp/s:30 mem_gb:52.82
|
| 26 |
+
Training: 4%|▍ | 1200/30000 [1:24:39<70:54:43, 8.86s/step]INFO 2026-07-21 15:14:05 ot_train.py:646 step:31K smpl:4M ep:90K epch:18.29 loss:0.017 grdn:0.151 lr:4.8e-05 updt_s:3.061 data_s:1.638 smp/s:27 mem_gb:52.82
|
| 27 |
+
Training: 5%|▍ | 1400/30000 [1:38:56<33:07:07, 4.17s/step]INFO 2026-07-21 15:28:22 ot_train.py:646 step:31K smpl:4M ep:91K epch:18.41 loss:0.018 grdn:0.153 lr:4.7e-05 updt_s:2.715 data_s:1.569 smp/s:30 mem_gb:52.82
|
| 28 |
+
Training: 5%|▌ | 1600/30000 [1:52:59<31:01:27, 3.93s/step]INFO 2026-07-21 15:42:26 ot_train.py:646 step:32K smpl:4M ep:91K epch:18.52 loss:0.017 grdn:0.151 lr:4.7e-05 updt_s:2.669 data_s:1.545 smp/s:30 mem_gb:52.82
|
| 29 |
+
Training: 6%|▌ | 1800/30000 [2:06:38<29:48:08, 3.80s/step]INFO 2026-07-21 15:56:04 ot_train.py:646 step:32K smpl:4M ep:92K epch:18.64 loss:0.017 grdn:0.148 lr:4.6e-05 updt_s:2.702 data_s:1.387 smp/s:31 mem_gb:52.74
|
| 30 |
+
Training: 7%|▋ | 2000/30000 [2:19:01<27:45:34, 3.57s/step]INFO 2026-07-21 16:08:28 ot_train.py:646 step:32K smpl:4M ep:92K epch:18.76 loss:0.016 grdn:0.141 lr:4.6e-05 updt_s:2.506 data_s:1.209 smp/s:34 mem_gb:52.82
|
| 31 |
+
Training: 7%|▋ | 2200/30000 [2:32:28<27:04:51, 3.51s/step]INFO 2026-07-21 16:21:54 ot_train.py:646 step:32K smpl:4M ep:93K epch:18.87 loss:0.016 grdn:0.139 lr:4.5e-05 updt_s:2.855 data_s:1.174 smp/s:32 mem_gb:52.82
|
| 32 |
+
Training: 8%|▊ | 2400/30000 [2:44:16<27:04:42, 3.53s/step]INFO 2026-07-21 16:33:42 ot_train.py:646 step:32K smpl:4M ep:94K epch:18.99 loss:0.016 grdn:0.139 lr:4.5e-05 updt_s:2.370 data_s:1.167 smp/s:36 mem_gb:52.82
|
| 33 |
+
Training: 9%|▊ | 2600/30000 [2:56:05<26:47:54, 3.52s/step]INFO 2026-07-21 16:45:31 ot_train.py:646 step:33K smpl:4M ep:94K epch:19.11 loss:0.016 grdn:0.135 lr:4.4e-05 updt_s:2.376 data_s:1.168 smp/s:36 mem_gb:52.82
|
| 34 |
+
Training: 9%|▉ | 2800/30000 [3:07:52<26:33:01, 3.51s/step]INFO 2026-07-21 16:57:19 ot_train.py:646 step:33K smpl:4M ep:95K epch:19.23 loss:0.016 grdn:0.137 lr:4.4e-05 updt_s:2.366 data_s:1.169 smp/s:36 mem_gb:52.74
|
| 35 |
+
Training: 10%|█ | 3000/30000 [3:19:38<26:47:36, 3.57s/step]INFO 2026-07-21 17:09:04 ot_train.py:646 step:33K smpl:4M ep:95K epch:19.34 loss:0.015 grdn:0.134 lr:4.3e-05 updt_s:2.369 data_s:1.156 smp/s:36 mem_gb:52.82
|
| 36 |
+
Training: 11%|█ | 3200/30000 [3:31:23<26:27:54, 3.56s/step]INFO 2026-07-21 17:20:49 ot_train.py:646 step:33K smpl:4M ep:96K epch:19.46 loss:0.015 grdn:0.127 lr:4.3e-05 updt_s:2.366 data_s:1.157 smp/s:36 mem_gb:52.82
|
| 37 |
+
Training: 11%|█▏ | 3400/30000 [3:43:08<26:10:59, 3.54s/step]INFO 2026-07-21 17:32:35 ot_train.py:646 step:33K smpl:4M ep:97K epch:19.58 loss:0.016 grdn:0.151 lr:4.2e-05 updt_s:2.366 data_s:1.160 smp/s:36 mem_gb:52.82
|
| 38 |
+
Training: 12%|█▏ | 3600/30000 [3:54:54<25:38:01, 3.50s/step]INFO 2026-07-21 17:44:21 ot_train.py:646 step:34K smpl:4M ep:97K epch:19.70 loss:0.015 grdn:0.137 lr:4.1e-05 updt_s:2.367 data_s:1.160 smp/s:36 mem_gb:52.82
|
| 39 |
+
Training: 13%|█▎ | 3800/30000 [4:06:40<25:38:25, 3.52s/step]INFO 2026-07-21 17:56:07 ot_train.py:646 step:34K smpl:4M ep:98K epch:19.81 loss:0.015 grdn:0.136 lr:4.1e-05 updt_s:2.368 data_s:1.161 smp/s:36 mem_gb:52.82
|
| 40 |
+
Training: 13%|█▎ | 4000/30000 [4:18:28<25:33:28, 3.54s/step]INFO 2026-07-21 18:07:54 ot_train.py:646 step:34K smpl:4M ep:98K epch:19.93 loss:0.015 grdn:0.130 lr:4.0e-05 updt_s:2.362 data_s:1.173 smp/s:36 mem_gb:52.74
|
| 41 |
+
Training: 14%|█▍ | 4200/30000 [4:30:14<25:15:12, 3.52s/step]INFO 2026-07-21 18:19:40 ot_train.py:646 step:34K smpl:4M ep:99K epch:20.05 loss:0.015 grdn:0.133 lr:4.0e-05 updt_s:2.369 data_s:1.160 smp/s:36 mem_gb:52.82
|
| 42 |
+
Training: 15%|█▍ | 4400/30000 [4:42:01<25:18:57, 3.56s/step]INFO 2026-07-21 18:31:28 ot_train.py:646 step:34K smpl:4M ep:99K epch:20.16 loss:0.015 grdn:0.135 lr:3.9e-05 updt_s:2.373 data_s:1.162 smp/s:36 mem_gb:52.82
|
| 43 |
+
Training: 15%|█▌ | 4600/30000 [4:53:47<24:52:31, 3.53s/step]INFO 2026-07-21 18:43:14 ot_train.py:646 step:35K smpl:4M ep:100K epch:20.28 loss:0.015 grdn:0.138 lr:3.9e-05 updt_s:2.365 data_s:1.161 smp/s:36 mem_gb:52.82
|
| 44 |
+
Training: 16%|█▌ | 4800/30000 [5:05:33<24:44:01, 3.53s/step]INFO 2026-07-21 18:54:59 ot_train.py:646 step:35K smpl:4M ep:101K epch:20.40 loss:0.015 grdn:0.128 lr:3.8e-05 updt_s:2.366 data_s:1.160 smp/s:36 mem_gb:52.82
|
| 45 |
+
Training: 17%|█▋ | 5000/30000 [5:17:46<26:46:22, 3.86s/step]INFO 2026-07-21 19:07:12 ot_train.py:646 step:35K smpl:4M ep:101K epch:20.52 loss:0.015 grdn:0.132 lr:3.8e-05 updt_s:2.485 data_s:1.178 smp/s:35 mem_gb:52.74
|
| 46 |
+
Training: 17%|█▋ | 5200/30000 [5:29:31<24:10:21, 3.51s/step]INFO 2026-07-21 19:18:57 ot_train.py:646 step:35K smpl:5M ep:102K epch:20.63 loss:0.014 grdn:0.125 lr:3.7e-05 updt_s:2.363 data_s:1.162 smp/s:36 mem_gb:52.82
|
| 47 |
+
Training: 18%|█▊ | 5400/30000 [5:41:17<24:07:04, 3.53s/step]INFO 2026-07-21 19:30:43 ot_train.py:646 step:35K smpl:5M ep:102K epch:20.75 loss:0.014 grdn:0.123 lr:3.7e-05 updt_s:2.363 data_s:1.165 smp/s:36 mem_gb:52.82
|
| 48 |
+
Training: 19%|█▊ | 5600/30000 [5:53:02<23:50:03, 3.52s/step]INFO 2026-07-21 19:42:29 ot_train.py:646 step:36K smpl:5M ep:103K epch:20.87 loss:0.014 grdn:0.118 lr:3.6e-05 updt_s:2.363 data_s:1.161 smp/s:36 mem_gb:52.82
|
| 49 |
+
Training: 19%|█▉ | 5800/30000 [6:04:47<23:39:12, 3.52s/step]INFO 2026-07-21 19:54:14 ot_train.py:646 step:36K smpl:5M ep:103K epch:20.98 loss:0.014 grdn:0.122 lr:3.6e-05 updt_s:2.364 data_s:1.159 smp/s:36 mem_gb:52.82
|
| 50 |
+
Training: 20%|██ | 6000/30000 [6:16:34<23:47:21, 3.57s/step]INFO 2026-07-21 20:06:01 ot_train.py:646 step:36K smpl:5M ep:104K epch:21.10 loss:0.014 grdn:0.128 lr:3.5e-05 updt_s:2.368 data_s:1.165 smp/s:36 mem_gb:52.82
|
| 51 |
+
Training: 21%|██ | 6200/30000 [6:28:22<23:31:47, 3.56s/step]INFO 2026-07-21 20:17:48 ot_train.py:646 step:36K smpl:5M ep:105K epch:21.22 loss:0.014 grdn:0.131 lr:3.5e-05 updt_s:2.363 data_s:1.173 smp/s:36 mem_gb:52.74
|
| 52 |
+
Training: 21%|██▏ | 6400/30000 [6:40:09<23:15:11, 3.55s/step]INFO 2026-07-21 20:29:36 ot_train.py:646 step:36K smpl:5M ep:105K epch:21.34 loss:0.013 grdn:0.121 lr:3.4e-05 updt_s:2.373 data_s:1.163 smp/s:36 mem_gb:52.82
|
| 53 |
+
Training: 22%|██▏ | 6600/30000 [6:51:54<22:52:03, 3.52s/step]INFO 2026-07-21 20:41:21 ot_train.py:646 step:37K smpl:5M ep:106K epch:21.45 loss:0.014 grdn:0.126 lr:3.4e-05 updt_s:2.362 data_s:1.162 smp/s:36 mem_gb:52.82
|
| 54 |
+
Training: 23%|██▎ | 6800/30000 [7:03:42<22:41:35, 3.52s/step]INFO 2026-07-21 20:53:08 ot_train.py:646 step:37K smpl:5M ep:106K epch:21.57 loss:0.013 grdn:0.125 lr:3.3e-05 updt_s:2.369 data_s:1.165 smp/s:36 mem_gb:52.82
|
| 55 |
+
Training: 23%|██▎ | 7000/30000 [7:15:27<22:28:42, 3.52s/step]INFO 2026-07-21 21:04:53 ot_train.py:646 step:37K smpl:5M ep:107K epch:21.69 loss:0.013 grdn:0.126 lr:3.3e-05 updt_s:2.363 data_s:1.163 smp/s:36 mem_gb:52.82
|
| 56 |
+
Training: 24%|██▍ | 7200/30000 [7:27:13<22:23:06, 3.53s/step]INFO 2026-07-21 21:16:39 ot_train.py:646 step:37K smpl:5M ep:108K epch:21.81 loss:0.013 grdn:0.123 lr:3.2e-05 updt_s:2.366 data_s:1.161 smp/s:36 mem_gb:52.82
|
| 57 |
+
Training: 25%|██▍ | 7400/30000 [7:39:05<22:13:41, 3.54s/step]INFO 2026-07-21 21:28:31 ot_train.py:646 step:37K smpl:5M ep:108K epch:21.92 loss:0.013 grdn:0.123 lr:3.2e-05 updt_s:2.374 data_s:1.182 smp/s:36 mem_gb:52.74
|
| 58 |
+
Training: 25%|██▌ | 7600/30000 [7:50:54<21:52:09, 3.51s/step]INFO 2026-07-21 21:40:20 ot_train.py:646 step:38K smpl:5M ep:109K epch:22.04 loss:0.013 grdn:0.128 lr:3.1e-05 updt_s:2.377 data_s:1.167 smp/s:36 mem_gb:52.82
|
| 59 |
+
Training: 26%|██▌ | 7800/30000 [8:02:42<21:39:11, 3.51s/step]INFO 2026-07-21 21:52:09 ot_train.py:646 step:38K smpl:5M ep:109K epch:22.16 loss:0.012 grdn:0.118 lr:3.1e-05 updt_s:2.374 data_s:1.167 smp/s:36 mem_gb:52.82
|
| 60 |
+
Training: 27%|██▋ | 8000/30000 [8:14:28<21:38:16, 3.54s/step]INFO 2026-07-21 22:03:54 ot_train.py:646 step:38K smpl:5M ep:110K epch:22.27 loss:0.013 grdn:0.123 lr:3.0e-05 updt_s:2.366 data_s:1.160 smp/s:36 mem_gb:52.82
|
| 61 |
+
Training: 27%|██▋ | 8200/30000 [8:26:15<21:18:33, 3.52s/step]INFO 2026-07-21 22:15:41 ot_train.py:646 step:38K smpl:5M ep:110K epch:22.39 loss:0.013 grdn:0.120 lr:3.0e-05 updt_s:2.369 data_s:1.163 smp/s:36 mem_gb:52.82
|
| 62 |
+
Training: 28%|██▊ | 8400/30000 [8:38:03<21:15:13, 3.54s/step]INFO 2026-07-21 22:27:29 ot_train.py:646 step:38K smpl:5M ep:111K epch:22.51 loss:0.012 grdn:0.116 lr:2.9e-05 updt_s:2.365 data_s:1.174 smp/s:36 mem_gb:52.74
|
| 63 |
+
Training: 29%|██▊ | 8600/30000 [8:49:51<21:03:20, 3.54s/step]INFO 2026-07-21 22:39:17 ot_train.py:646 step:39K smpl:5M ep:112K epch:22.63 loss:0.012 grdn:0.115 lr:2.9e-05 updt_s:2.371 data_s:1.166 smp/s:36 mem_gb:52.82
|
| 64 |
+
Training: 29%|██▉ | 8800/30000 [9:01:47<20:55:58, 3.55s/step]INFO 2026-07-21 22:51:14 ot_train.py:646 step:39K smpl:5M ep:112K epch:22.74 loss:0.012 grdn:0.113 lr:2.8e-05 updt_s:2.400 data_s:1.180 smp/s:36 mem_gb:52.82
|
| 65 |
+
Training: 30%|███ | 9000/30000 [9:13:41<20:38:55, 3.54s/step]INFO 2026-07-21 23:03:07 ot_train.py:646 step:39K smpl:5M ep:113K epch:22.86 loss:0.012 grdn:0.115 lr:2.8e-05 updt_s:2.390 data_s:1.175 smp/s:36 mem_gb:52.82
|
| 66 |
+
Training: 31%|███ | 9200/30000 [9:25:35<20:31:01, 3.55s/step]INFO 2026-07-21 23:15:02 ot_train.py:646 step:39K smpl:5M ep:113K epch:22.98 loss:0.012 grdn:0.117 lr:2.7e-05 updt_s:2.391 data_s:1.181 smp/s:36 mem_gb:52.82
|
| 67 |
+
Training: 31%|███▏ | 9400/30000 [9:37:30<20:30:36, 3.58s/step]INFO 2026-07-21 23:26:57 ot_train.py:646 step:39K smpl:5M ep:114K epch:23.10 loss:0.012 grdn:0.114 lr:2.7e-05 updt_s:2.388 data_s:1.185 smp/s:36 mem_gb:52.82
|
| 68 |
+
Training: 32%|███▏ | 9600/30000 [9:49:30<20:18:38, 3.58s/step]INFO 2026-07-21 23:38:56 ot_train.py:646 step:40K smpl:5M ep:114K epch:23.21 loss:0.011 grdn:0.112 lr:2.7e-05 updt_s:2.394 data_s:1.200 smp/s:36 mem_gb:52.74
|
| 69 |
+
Training: 33%|███▎ | 9800/30000 [10:01:26<20:15:50, 3.61s/step]INFO 2026-07-21 23:50:53 ot_train.py:646 step:40K smpl:5M ep:115K epch:23.33 loss:0.011 grdn:0.109 lr:2.6e-05 updt_s:2.399 data_s:1.183 smp/s:36 mem_gb:52.82
|
| 70 |
+
Training: 33%|███▎ | 10000/30000 [10:13:24<19:42:11, 3.55s/step]INFO 2026-07-22 00:02:50 ot_train.py:646 step:40K smpl:5M ep:116K epch:23.45 loss:0.012 grdn:0.111 lr:2.6e-05 updt_s:2.398 data_s:1.186 smp/s:36 mem_gb:52.82
|
| 71 |
+
Training: 34%|███▍ | 10200/30000 [10:25:22<19:44:21, 3.59s/step]INFO 2026-07-22 00:14:48 ot_train.py:646 step:40K smpl:5M ep:116K epch:23.56 loss:0.011 grdn:0.111 lr:2.5e-05 updt_s:2.399 data_s:1.188 smp/s:36 mem_gb:52.82
|
| 72 |
+
Training: 35%|███▍ | 10400/30000 [10:37:17<19:20:13, 3.55s/step]INFO 2026-07-22 00:26:43 ot_train.py:646 step:40K smpl:5M ep:117K epch:23.68 loss:0.011 grdn:0.114 lr:2.5e-05 updt_s:2.388 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 73 |
+
Training: 35%|███▌ | 10600/30000 [10:49:15<19:19:32, 3.59s/step]INFO 2026-07-22 00:38:41 ot_train.py:646 step:41K smpl:5M ep:117K epch:23.80 loss:0.011 grdn:0.110 lr:2.4e-05 updt_s:2.397 data_s:1.190 smp/s:36 mem_gb:52.82
|
| 74 |
+
Training: 36%|███▌ | 10800/30000 [11:01:12<19:11:04, 3.60s/step]INFO 2026-07-22 00:50:39 ot_train.py:646 step:41K smpl:5M ep:118K epch:23.92 loss:0.011 grdn:0.115 lr:2.4e-05 updt_s:2.391 data_s:1.193 smp/s:36 mem_gb:52.74
|
| 75 |
+
Training: 37%|███▋ | 11000/30000 [11:13:11<18:51:03, 3.57s/step]INFO 2026-07-22 01:02:37 ot_train.py:646 step:41K smpl:5M ep:118K epch:24.03 loss:0.011 grdn:0.114 lr:2.3e-05 updt_s:2.402 data_s:1.188 smp/s:36 mem_gb:52.82
|
| 76 |
+
Training: 37%|███▋ | 11200/30000 [11:25:06<18:47:40, 3.60s/step]INFO 2026-07-22 01:14:33 ot_train.py:646 step:41K smpl:5M ep:119K epch:24.15 loss:0.011 grdn:0.114 lr:2.3e-05 updt_s:2.394 data_s:1.181 smp/s:36 mem_gb:52.82
|
| 77 |
+
Training: 38%|███▊ | 11400/30000 [11:37:04<18:25:03, 3.56s/step]INFO 2026-07-22 01:26:30 ot_train.py:646 step:41K smpl:5M ep:120K epch:24.27 loss:0.011 grdn:0.110 lr:2.2e-05 updt_s:2.398 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 78 |
+
Training: 39%|███▊ | 11600/30000 [11:49:01<18:17:06, 3.58s/step]INFO 2026-07-22 01:38:27 ot_train.py:646 step:42K smpl:5M ep:120K epch:24.38 loss:0.011 grdn:0.106 lr:2.2e-05 updt_s:2.397 data_s:1.186 smp/s:36 mem_gb:52.82
|
| 79 |
+
Training: 39%|███▉ | 11800/30000 [12:00:59<18:00:56, 3.56s/step]INFO 2026-07-22 01:50:25 ot_train.py:646 step:42K smpl:5M ep:121K epch:24.50 loss:0.010 grdn:0.107 lr:2.2e-05 updt_s:2.391 data_s:1.199 smp/s:36 mem_gb:52.74
|
| 80 |
+
Training: 40%|████ | 12000/30000 [12:12:55<17:54:30, 3.58s/step]INFO 2026-07-22 02:02:22 ot_train.py:646 step:42K smpl:5M ep:121K epch:24.62 loss:0.010 grdn:0.104 lr:2.1e-05 updt_s:2.392 data_s:1.188 smp/s:36 mem_gb:52.82
|
| 81 |
+
Training: 41%|████ | 12200/30000 [12:24:52<17:41:01, 3.58s/step]INFO 2026-07-22 02:14:18 ot_train.py:646 step:42K smpl:5M ep:122K epch:24.74 loss:0.010 grdn:0.110 lr:2.1e-05 updt_s:2.393 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 82 |
+
Training: 41%|████▏ | 12400/30000 [12:36:48<17:29:25, 3.58s/step]INFO 2026-07-22 02:26:15 ot_train.py:646 step:42K smpl:5M ep:123K epch:24.85 loss:0.010 grdn:0.101 lr:2.0e-05 updt_s:2.395 data_s:1.185 smp/s:36 mem_gb:52.82
|
| 83 |
+
Training: 42%|████▏ | 12600/30000 [12:48:44<17:18:31, 3.58s/step]INFO 2026-07-22 02:38:10 ot_train.py:646 step:43K smpl:5M ep:123K epch:24.97 loss:0.010 grdn:0.102 lr:2.0e-05 updt_s:2.389 data_s:1.186 smp/s:36 mem_gb:52.82
|
| 84 |
+
Training: 43%|████▎ | 12800/30000 [13:00:39<17:00:11, 3.56s/step]INFO 2026-07-22 02:50:05 ot_train.py:646 step:43K smpl:5M ep:124K epch:25.09 loss:0.010 grdn:0.103 lr:1.9e-05 updt_s:2.391 data_s:1.184 smp/s:36 mem_gb:52.82
|
| 85 |
+
Training: 43%|████▎ | 13000/30000 [13:12:34<16:48:12, 3.56s/step]INFO 2026-07-22 03:02:00 ot_train.py:646 step:43K smpl:6M ep:124K epch:25.21 loss:0.010 grdn:0.101 lr:1.9e-05 updt_s:2.381 data_s:1.192 smp/s:36 mem_gb:52.74
|
| 86 |
+
Training: 44%|████▍ | 13200/30000 [13:24:32<16:40:21, 3.57s/step]INFO 2026-07-22 03:13:58 ot_train.py:646 step:43K smpl:6M ep:125K epch:25.32 loss:0.010 grdn:0.100 lr:1.9e-05 updt_s:2.399 data_s:1.188 smp/s:36 mem_gb:52.82
|
| 87 |
+
Training: 45%|████▍ | 13400/30000 [13:36:29<16:41:16, 3.62s/step]INFO 2026-07-22 03:25:56 ot_train.py:646 step:43K smpl:6M ep:125K epch:25.44 loss:0.009 grdn:0.101 lr:1.8e-05 updt_s:2.394 data_s:1.191 smp/s:36 mem_gb:52.82
|
| 88 |
+
Training: 45%|████▌ | 13600/30000 [13:48:28<16:17:16, 3.58s/step]INFO 2026-07-22 03:37:55 ot_train.py:646 step:44K smpl:6M ep:126K epch:25.56 loss:0.010 grdn:0.104 lr:1.8e-05 updt_s:2.405 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 89 |
+
Training: 46%|████▌ | 13800/30000 [14:00:24<16:05:09, 3.57s/step]INFO 2026-07-22 03:49:50 ot_train.py:646 step:44K smpl:6M ep:127K epch:25.67 loss:0.010 grdn:0.102 lr:1.7e-05 updt_s:2.393 data_s:1.183 smp/s:36 mem_gb:52.82
|
| 90 |
+
Training: 47%|████▋ | 14000/30000 [14:12:20<16:00:20, 3.60s/step]INFO 2026-07-22 04:01:47 ot_train.py:646 step:44K smpl:6M ep:127K epch:25.79 loss:0.010 grdn:0.103 lr:1.7e-05 updt_s:2.393 data_s:1.189 smp/s:36 mem_gb:52.82
|
| 91 |
+
Training: 47%|████▋ | 14200/30000 [14:24:17<15:43:25, 3.58s/step]INFO 2026-07-22 04:13:43 ot_train.py:646 step:44K smpl:6M ep:128K epch:25.91 loss:0.009 grdn:0.099 lr:1.7e-05 updt_s:2.386 data_s:1.195 smp/s:36 mem_gb:52.74
|
| 92 |
+
Training: 48%|████▊ | 14400/30000 [14:36:13<15:45:26, 3.64s/step]INFO 2026-07-22 04:25:39 ot_train.py:646 step:44K smpl:6M ep:128K epch:26.03 loss:0.009 grdn:0.101 lr:1.6e-05 updt_s:2.389 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 93 |
+
Training: 49%|████▊ | 14600/30000 [14:48:08<15:16:05, 3.57s/step]INFO 2026-07-22 04:37:34 ot_train.py:646 step:45K smpl:6M ep:129K epch:26.14 loss:0.010 grdn:0.104 lr:1.6e-05 updt_s:2.389 data_s:1.184 smp/s:36 mem_gb:52.82
|
| 94 |
+
Training: 49%|████▉ | 14800/30000 [15:00:04<15:03:40, 3.57s/step]INFO 2026-07-22 04:49:30 ot_train.py:646 step:45K smpl:6M ep:129K epch:26.26 loss:0.009 grdn:0.100 lr:1.5e-05 updt_s:2.392 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 95 |
+
Training: 50%|█████ | 15000/30000 [15:12:00<14:47:45, 3.55s/step]INFO 2026-07-22 05:01:26 ot_train.py:646 step:45K smpl:6M ep:130K epch:26.38 loss:0.009 grdn:0.097 lr:1.5e-05 updt_s:2.391 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 96 |
+
Training: 51%|█████ | 15200/30000 [15:23:57<14:41:21, 3.57s/step]INFO 2026-07-22 05:13:23 ot_train.py:646 step:45K smpl:6M ep:131K epch:26.49 loss:0.009 grdn:0.100 lr:1.5e-05 updt_s:2.388 data_s:1.197 smp/s:36 mem_gb:52.74
|
| 97 |
+
Training: 51%|█████▏ | 15400/30000 [15:35:53<14:31:18, 3.58s/step]INFO 2026-07-22 05:25:19 ot_train.py:646 step:45K smpl:6M ep:131K epch:26.61 loss:0.009 grdn:0.098 lr:1.4e-05 updt_s:2.392 data_s:1.183 smp/s:36 mem_gb:52.82
|
| 98 |
+
Training: 52%|█████▏ | 15600/30000 [15:47:49<14:14:43, 3.56s/step]INFO 2026-07-22 05:37:15 ot_train.py:646 step:46K smpl:6M ep:132K epch:26.73 loss:0.009 grdn:0.098 lr:1.4e-05 updt_s:2.388 data_s:1.189 smp/s:36 mem_gb:52.82
|
| 99 |
+
Training: 53%|█████▎ | 15800/30000 [15:59:45<14:05:48, 3.57s/step]INFO 2026-07-22 05:49:11 ot_train.py:646 step:46K smpl:6M ep:132K epch:26.85 loss:0.009 grdn:0.094 lr:1.4e-05 updt_s:2.398 data_s:1.181 smp/s:36 mem_gb:52.82
|
| 100 |
+
Training: 53%|█████▎ | 16000/30000 [16:11:39<13:52:06, 3.57s/step]INFO 2026-07-22 06:01:06 ot_train.py:646 step:46K smpl:6M ep:133K epch:26.96 loss:0.009 grdn:0.095 lr:1.3e-05 updt_s:2.391 data_s:1.180 smp/s:36 mem_gb:52.82
|
| 101 |
+
Training: 54%|█████▍ | 16200/30000 [16:23:34<13:36:13, 3.55s/step]INFO 2026-07-22 06:13:01 ot_train.py:646 step:46K smpl:6M ep:134K epch:27.08 loss:0.009 grdn:0.097 lr:1.3e-05 updt_s:2.393 data_s:1.179 smp/s:36 mem_gb:52.82
|
| 102 |
+
Training: 55%|█████▍ | 16400/30000 [16:35:32<13:28:47, 3.57s/step]INFO 2026-07-22 06:24:58 ot_train.py:646 step:46K smpl:6M ep:134K epch:27.20 loss:0.009 grdn:0.098 lr:1.3e-05 updt_s:2.396 data_s:1.190 smp/s:36 mem_gb:52.74
|
| 103 |
+
Training: 55%|█████▌ | 16600/30000 [16:47:28<13:13:32, 3.55s/step]INFO 2026-07-22 06:36:55 ot_train.py:646 step:47K smpl:6M ep:135K epch:27.32 loss:0.008 grdn:0.099 lr:1.2e-05 updt_s:2.393 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 104 |
+
Training: 56%|█████▌ | 16800/30000 [16:59:27<13:05:06, 3.57s/step]INFO 2026-07-22 06:48:53 ot_train.py:646 step:47K smpl:6M ep:135K epch:27.43 loss:0.008 grdn:0.095 lr:1.2e-05 updt_s:2.400 data_s:1.190 smp/s:36 mem_gb:52.82
|
| 105 |
+
Training: 57%|█████▋ | 17000/30000 [17:11:23<12:51:05, 3.56s/step]INFO 2026-07-22 07:00:49 ot_train.py:646 step:47K smpl:6M ep:136K epch:27.55 loss:0.008 grdn:0.096 lr:1.1e-05 updt_s:2.395 data_s:1.183 smp/s:36 mem_gb:52.82
|
| 106 |
+
Training: 57%|█████▋ | 17200/30000 [17:23:19<12:40:50, 3.57s/step]INFO 2026-07-22 07:12:45 ot_train.py:646 step:47K smpl:6M ep:136K epch:27.67 loss:0.008 grdn:0.096 lr:1.1e-05 updt_s:2.396 data_s:1.181 smp/s:36 mem_gb:52.82
|
| 107 |
+
Training: 58%|█████▊ | 17400/30000 [17:35:15<12:26:44, 3.56s/step]INFO 2026-07-22 07:24:42 ot_train.py:646 step:47K smpl:6M ep:137K epch:27.78 loss:0.008 grdn:0.094 lr:1.1e-05 updt_s:2.398 data_s:1.182 smp/s:36 mem_gb:52.82
|
| 108 |
+
Training: 59%|█████▊ | 17600/30000 [17:47:12<12:16:17, 3.56s/step]INFO 2026-07-22 07:36:39 ot_train.py:646 step:48K smpl:6M ep:138K epch:27.90 loss:0.008 grdn:0.092 lr:1.0e-05 updt_s:2.393 data_s:1.190 smp/s:36 mem_gb:52.74
|
| 109 |
+
Training: 59%|█████▉ | 17800/30000 [17:59:14<12:14:04, 3.61s/step]INFO 2026-07-22 07:48:41 ot_train.py:646 step:48K smpl:6M ep:138K epch:28.02 loss:0.008 grdn:0.091 lr:1.0e-05 updt_s:2.416 data_s:1.192 smp/s:35 mem_gb:52.82
|
| 110 |
+
Training: 60%|██████ | 18000/30000 [18:11:22<11:58:51, 3.59s/step]INFO 2026-07-22 08:00:48 ot_train.py:646 step:48K smpl:6M ep:139K epch:28.14 loss:0.008 grdn:0.094 lr:9.9e-06 updt_s:2.434 data_s:1.200 smp/s:35 mem_gb:52.82
|
| 111 |
+
Training: 61%|██████ | 18200/30000 [18:23:26<11:50:07, 3.61s/step]INFO 2026-07-22 08:12:52 ot_train.py:646 step:48K smpl:6M ep:139K epch:28.25 loss:0.008 grdn:0.090 lr:9.5e-06 updt_s:2.423 data_s:1.197 smp/s:35 mem_gb:52.82
|
| 112 |
+
Training: 61%|██████▏ | 18400/30000 [18:35:27<11:32:17, 3.58s/step]INFO 2026-07-22 08:24:53 ot_train.py:646 step:48K smpl:6M ep:140K epch:28.37 loss:0.008 grdn:0.093 lr:9.2e-06 updt_s:2.408 data_s:1.195 smp/s:36 mem_gb:52.82
|
| 113 |
+
Training: 62%|██████▏ | 18600/30000 [18:47:29<11:24:00, 3.60s/step]INFO 2026-07-22 08:36:55 ot_train.py:646 step:49K smpl:6M ep:140K epch:28.49 loss:0.008 grdn:0.094 lr:8.9e-06 updt_s:2.403 data_s:1.204 smp/s:35 mem_gb:52.74
|
| 114 |
+
Training: 63%|██████▎ | 18800/30000 [18:59:31<11:09:57, 3.59s/step]INFO 2026-07-22 08:48:57 ot_train.py:646 step:49K smpl:6M ep:141K epch:28.61 loss:0.008 grdn:0.091 lr:8.6e-06 updt_s:2.413 data_s:1.195 smp/s:35 mem_gb:52.82
|
| 115 |
+
Training: 63%|██████▎ | 19000/30000 [19:11:31<10:51:41, 3.55s/step]INFO 2026-07-22 09:00:57 ot_train.py:646 step:49K smpl:6M ep:142K epch:28.72 loss:0.007 grdn:0.090 lr:8.3e-06 updt_s:2.407 data_s:1.191 smp/s:36 mem_gb:52.82
|
| 116 |
+
Training: 64%|██████▍ | 19200/30000 [19:23:29<10:59:28, 3.66s/step]INFO 2026-07-22 09:12:55 ot_train.py:646 step:49K smpl:6M ep:142K epch:28.84 loss:0.008 grdn:0.092 lr:8.1e-06 updt_s:2.402 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 117 |
+
Training: 65%|██████▍ | 19400/30000 [19:35:29<10:35:52, 3.60s/step]INFO 2026-07-22 09:24:56 ot_train.py:646 step:49K smpl:6M ep:143K epch:28.96 loss:0.008 grdn:0.092 lr:7.8e-06 updt_s:2.408 data_s:1.191 smp/s:36 mem_gb:52.82
|
| 118 |
+
Training: 65%|██████▌ | 19600/30000 [19:47:27<10:30:34, 3.64s/step]INFO 2026-07-22 09:36:53 ot_train.py:646 step:50K smpl:6M ep:143K epch:29.07 loss:0.008 grdn:0.091 lr:7.5e-06 updt_s:2.400 data_s:1.185 smp/s:36 mem_gb:52.82
|
| 119 |
+
Training: 66%|██████▌ | 19800/30000 [19:59:26<10:11:33, 3.60s/step]INFO 2026-07-22 09:48:53 ot_train.py:646 step:50K smpl:6M ep:144K epch:29.19 loss:0.007 grdn:0.092 lr:7.2e-06 updt_s:2.399 data_s:1.198 smp/s:36 mem_gb:52.74
|
| 120 |
+
Training: 67%|██████▋ | 20000/30000 [20:11:23<9:56:36, 3.58s/step]INFO 2026-07-22 10:00:49 ot_train.py:646 step:50K smpl:6M ep:144K epch:29.31 loss:0.007 grdn:0.088 lr:6.9e-06 updt_s:2.397 data_s:1.182 smp/s:36 mem_gb:52.82
|
| 121 |
+
Training: 67%|██████▋ | 20200/30000 [20:23:21<9:53:22, 3.63s/step]INFO 2026-07-22 10:12:47 ot_train.py:646 step:50K smpl:6M ep:145K epch:29.43 loss:0.007 grdn:0.091 lr:6.7e-06 updt_s:2.402 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 122 |
+
Training: 68%|██████▊ | 20400/30000 [20:35:19<9:40:03, 3.63s/step]INFO 2026-07-22 10:24:45 ot_train.py:646 step:50K smpl:6M ep:146K epch:29.54 loss:0.007 grdn:0.088 lr:6.4e-06 updt_s:2.402 data_s:1.185 smp/s:36 mem_gb:52.82
|
| 123 |
+
Training: 69%|██████▊ | 20600/30000 [20:47:16<9:21:56, 3.59s/step]INFO 2026-07-22 10:36:42 ot_train.py:646 step:51K smpl:6M ep:146K epch:29.66 loss:0.007 grdn:0.088 lr:6.2e-06 updt_s:2.397 data_s:1.186 smp/s:36 mem_gb:52.82
|
| 124 |
+
Training: 69%|██████▉ | 20800/30000 [20:59:15<9:08:23, 3.58s/step]INFO 2026-07-22 10:48:42 ot_train.py:646 step:51K smpl:7M ep:147K epch:29.78 loss:0.007 grdn:0.089 lr:5.9e-06 updt_s:2.407 data_s:1.190 smp/s:36 mem_gb:52.82
|
| 125 |
+
Training: 70%|███████ | 21000/30000 [21:11:12<8:55:42, 3.57s/step]INFO 2026-07-22 11:00:38 ot_train.py:646 step:51K smpl:7M ep:147K epch:29.89 loss:0.007 grdn:0.089 lr:5.7e-06 updt_s:2.389 data_s:1.191 smp/s:36 mem_gb:52.74
|
| 126 |
+
Training: 71%|███████ | 21200/30000 [21:23:06<8:43:50, 3.57s/step]INFO 2026-07-22 11:12:32 ot_train.py:646 step:51K smpl:7M ep:148K epch:30.01 loss:0.007 grdn:0.086 lr:5.4e-06 updt_s:2.391 data_s:1.178 smp/s:36 mem_gb:52.82
|
| 127 |
+
Training: 71%|███████▏ | 21400/30000 [21:35:01<8:32:07, 3.57s/step]INFO 2026-07-22 11:24:28 ot_train.py:646 step:51K smpl:7M ep:149K epch:30.13 loss:0.007 grdn:0.088 lr:5.2e-06 updt_s:2.395 data_s:1.180 smp/s:36 mem_gb:52.82
|
| 128 |
+
Training: 72%|███████▏ | 21600/30000 [21:47:01<8:23:52, 3.60s/step]INFO 2026-07-22 11:36:27 ot_train.py:646 step:52K smpl:7M ep:149K epch:30.25 loss:0.007 grdn:0.087 lr:5.0e-06 updt_s:2.408 data_s:1.186 smp/s:36 mem_gb:52.82
|
| 129 |
+
Training: 73%|███████▎ | 21800/30000 [21:59:02<8:19:31, 3.66s/step]INFO 2026-07-22 11:48:28 ot_train.py:646 step:52K smpl:7M ep:150K epch:30.36 loss:0.007 grdn:0.086 lr:4.7e-06 updt_s:2.413 data_s:1.192 smp/s:36 mem_gb:52.82
|
| 130 |
+
Training: 73%|███████▎ | 22000/30000 [22:11:04<8:05:02, 3.64s/step]INFO 2026-07-22 12:00:30 ot_train.py:646 step:52K smpl:7M ep:150K epch:30.48 loss:0.007 grdn:0.086 lr:4.5e-06 updt_s:2.403 data_s:1.204 smp/s:35 mem_gb:52.74
|
| 131 |
+
Training: 74%|███████▍ | 22200/30000 [22:23:02<7:46:59, 3.59s/step]INFO 2026-07-22 12:12:29 ot_train.py:646 step:52K smpl:7M ep:151K epch:30.60 loss:0.007 grdn:0.084 lr:4.3e-06 updt_s:2.405 data_s:1.185 smp/s:36 mem_gb:52.82
|
| 132 |
+
Training: 75%|███████▍ | 22400/30000 [22:35:04<7:49:51, 3.71s/step]INFO 2026-07-22 12:24:31 ot_train.py:646 step:52K smpl:7M ep:151K epch:30.72 loss:0.007 grdn:0.085 lr:4.1e-06 updt_s:2.415 data_s:1.192 smp/s:35 mem_gb:52.82
|
| 133 |
+
Training: 75%|███████▌ | 22600/30000 [22:47:06<7:24:21, 3.60s/step]INFO 2026-07-22 12:36:32 ot_train.py:646 step:53K smpl:7M ep:152K epch:30.83 loss:0.007 grdn:0.088 lr:3.9e-06 updt_s:2.410 data_s:1.194 smp/s:36 mem_gb:52.82
|
| 134 |
+
Training: 76%|███████▌ | 22800/30000 [22:59:09<7:10:00, 3.58s/step]INFO 2026-07-22 12:48:36 ot_train.py:646 step:53K smpl:7M ep:153K epch:30.95 loss:0.007 grdn:0.085 lr:3.7e-06 updt_s:2.418 data_s:1.199 smp/s:35 mem_gb:52.82
|
| 135 |
+
Training: 77%|███████▋ | 23000/30000 [23:11:08<6:57:24, 3.58s/step]INFO 2026-07-22 13:00:34 ot_train.py:646 step:53K smpl:7M ep:153K epch:31.07 loss:0.007 grdn:0.086 lr:3.5e-06 updt_s:2.405 data_s:1.187 smp/s:36 mem_gb:52.82
|
| 136 |
+
Training: 77%|███████▋ | 23200/30000 [23:23:09<6:50:23, 3.62s/step]INFO 2026-07-22 13:12:35 ot_train.py:646 step:53K smpl:7M ep:154K epch:31.18 loss:0.007 grdn:0.087 lr:3.3e-06 updt_s:2.397 data_s:1.205 smp/s:36 mem_gb:52.74
|
| 137 |
+
Training: 78%|███████▊ | 23400/30000 [23:35:10<6:36:40, 3.61s/step]INFO 2026-07-22 13:24:36 ot_train.py:646 step:53K smpl:7M ep:154K epch:31.30 loss:0.007 grdn:0.085 lr:3.1e-06 updt_s:2.408 data_s:1.194 smp/s:36 mem_gb:52.82
|
| 138 |
+
Training: 79%|███████▊ | 23600/30000 [23:47:10<6:24:09, 3.60s/step]INFO 2026-07-22 13:36:37 ot_train.py:646 step:54K smpl:7M ep:155K epch:31.42 loss:0.007 grdn:0.085 lr:2.9e-06 updt_s:2.408 data_s:1.193 smp/s:36 mem_gb:52.82
|
| 139 |
+
Training: 79%|███████▉ | 23800/30000 [23:59:10<6:08:30, 3.57s/step]INFO 2026-07-22 13:48:36 ot_train.py:646 step:54K smpl:7M ep:155K epch:31.54 loss:0.007 grdn:0.087 lr:2.7e-06 updt_s:2.408 data_s:1.188 smp/s:36 mem_gb:52.82
|
| 140 |
+
Training: 80%|████████ | 24000/30000 [24:15:17<8:06:08, 4.86s/step]INFO 2026-07-22 14:04:43 ot_train.py:646 step:54K smpl:7M ep:156K epch:31.65 loss:0.007 grdn:0.084 lr:2.6e-06 updt_s:2.893 data_s:1.938 smp/s:26 mem_gb:52.82
|
| 141 |
+
Training: 81%|████████ | 24200/30000 [24:27:31<6:08:21, 3.81s/step]INFO 2026-07-22 14:16:57 ot_train.py:646 step:54K smpl:7M ep:157K epch:31.77 loss:0.007 grdn:0.085 lr:2.4e-06 updt_s:2.425 data_s:1.241 smp/s:35 mem_gb:52.74
|
| 142 |
+
Training: 81%|████████▏ | 24400/30000 [24:39:38<5:34:44, 3.59s/step]INFO 2026-07-22 14:29:04 ot_train.py:646 step:54K smpl:7M ep:157K epch:31.89 loss:0.006 grdn:0.082 lr:2.2e-06 updt_s:2.430 data_s:1.205 smp/s:35 mem_gb:52.82
|
| 143 |
+
Training: 82%|████████▏ | 24600/30000 [24:51:53<5:19:09, 3.55s/step]INFO 2026-07-22 14:41:19 ot_train.py:646 step:55K smpl:7M ep:158K epch:32.00 loss:0.007 grdn:0.081 lr:2.1e-06 updt_s:2.418 data_s:1.254 smp/s:35 mem_gb:52.82
|
| 144 |
+
Training: 83%|████████▎ | 24800/30000 [25:03:56<5:12:40, 3.61s/step]INFO 2026-07-22 14:53:23 ot_train.py:646 step:55K smpl:7M ep:158K epch:32.12 loss:0.006 grdn:0.083 lr:1.9e-06 updt_s:2.415 data_s:1.200 smp/s:35 mem_gb:52.82
|
| 145 |
+
Training: 83%|████████▎ | 25000/30000 [25:15:58<4:56:29, 3.56s/step]INFO 2026-07-22 15:05:24 ot_train.py:646 step:55K smpl:7M ep:159K epch:32.24 loss:0.006 grdn:0.082 lr:1.8e-06 updt_s:2.404 data_s:1.201 smp/s:36 mem_gb:52.82
|
| 146 |
+
Training: 84%|████████▍ | 25200/30000 [25:28:10<4:50:51, 3.64s/step]INFO 2026-07-22 15:17:36 ot_train.py:646 step:55K smpl:7M ep:160K epch:32.36 loss:0.006 grdn:0.086 lr:1.7e-06 updt_s:2.429 data_s:1.228 smp/s:35 mem_gb:52.82
|
| 147 |
+
Training: 85%|████████▍ | 25400/30000 [25:40:11<4:35:19, 3.59s/step]INFO 2026-07-22 15:29:38 ot_train.py:646 step:55K smpl:7M ep:160K epch:32.47 loss:0.006 grdn:0.082 lr:1.5e-06 updt_s:2.401 data_s:1.204 smp/s:36 mem_gb:52.74
|
| 148 |
+
Training: 85%|████████▌ | 25600/30000 [25:52:12<4:44:40, 3.88s/step]INFO 2026-07-22 15:41:39 ot_train.py:646 step:56K smpl:7M ep:161K epch:32.59 loss:0.007 grdn:0.083 lr:1.4e-06 updt_s:2.408 data_s:1.194 smp/s:36 mem_gb:52.82
|
| 149 |
+
Training: 86%|████████▌ | 25800/30000 [26:04:42<4:26:42, 3.81s/step]INFO 2026-07-22 15:54:09 ot_train.py:646 step:56K smpl:7M ep:161K epch:32.71 loss:0.006 grdn:0.082 lr:1.3e-06 updt_s:2.463 data_s:1.285 smp/s:34 mem_gb:52.82
|
| 150 |
+
Training: 87%|████████▋ | 26000/30000 [26:19:53<4:01:15, 3.62s/step]INFO 2026-07-22 16:09:20 ot_train.py:646 step:56K smpl:7M ep:162K epch:32.83 loss:0.006 grdn:0.082 lr:1.2e-06 updt_s:2.757 data_s:1.795 smp/s:28 mem_gb:52.82
|
| 151 |
+
Training: 87%|████████▋ | 26200/30000 [26:31:53<3:49:05, 3.62s/step]INFO 2026-07-22 16:21:19 ot_train.py:646 step:56K smpl:7M ep:162K epch:32.94 loss:0.006 grdn:0.080 lr:1.1e-06 updt_s:2.410 data_s:1.186 smp/s:36 mem_gb:52.82
|
| 152 |
+
Training: 88%|████████▊ | 26400/30000 [26:43:46<3:32:49, 3.55s/step]INFO 2026-07-22 16:33:12 ot_train.py:646 step:56K smpl:7M ep:163K epch:33.06 loss:0.006 grdn:0.079 lr:9.5e-07 updt_s:2.383 data_s:1.180 smp/s:36 mem_gb:52.82
|
| 153 |
+
Training: 89%|████████▊ | 26600/30000 [26:55:38<3:21:37, 3.56s/step]INFO 2026-07-22 16:45:04 ot_train.py:646 step:57K smpl:7M ep:164K epch:33.18 loss:0.006 grdn:0.082 lr:8.5e-07 updt_s:2.378 data_s:1.179 smp/s:36 mem_gb:52.74
|
| 154 |
+
Training: 89%|████████▉ | 26800/30000 [27:07:31<3:08:16, 3.53s/step]INFO 2026-07-22 16:56:57 ot_train.py:646 step:57K smpl:7M ep:164K epch:33.29 loss:0.006 grdn:0.081 lr:7.6e-07 updt_s:2.386 data_s:1.176 smp/s:36 mem_gb:52.82
|
| 155 |
+
Training: 90%|█████████ | 27000/30000 [27:19:16<2:55:35, 3.51s/step]INFO 2026-07-22 17:08:43 ot_train.py:646 step:57K smpl:7M ep:165K epch:33.41 loss:0.006 grdn:0.080 lr:6.7e-07 updt_s:2.367 data_s:1.161 smp/s:36 mem_gb:52.82
|
| 156 |
+
Training: 91%|█████████ | 27200/30000 [27:31:05<2:45:57, 3.56s/step]INFO 2026-07-22 17:20:32 ot_train.py:646 step:57K smpl:7M ep:165K epch:33.53 loss:0.006 grdn:0.080 lr:5.9e-07 updt_s:2.377 data_s:1.166 smp/s:36 mem_gb:52.82
|
| 157 |
+
Training: 91%|█████████▏| 27400/30000 [27:42:58<2:35:53, 3.60s/step]INFO 2026-07-22 17:32:24 ot_train.py:646 step:57K smpl:7M ep:166K epch:33.65 loss:0.006 grdn:0.082 lr:5.1e-07 updt_s:2.389 data_s:1.171 smp/s:36 mem_gb:52.82
|
| 158 |
+
Training: 92%|█████████▏| 27600/30000 [27:54:57<2:20:51, 3.52s/step]INFO 2026-07-22 17:44:23 ot_train.py:646 step:58K smpl:7M ep:166K epch:33.76 loss:0.006 grdn:0.079 lr:4.4e-07 updt_s:2.395 data_s:1.196 smp/s:36 mem_gb:52.74
|
| 159 |
+
Training: 93%|█████████▎| 27800/30000 [28:06:49<2:11:05, 3.58s/step]INFO 2026-07-22 17:56:15 ot_train.py:646 step:58K smpl:7M ep:167K epch:33.88 loss:0.006 grdn:0.079 lr:3.7e-07 updt_s:2.385 data_s:1.175 smp/s:36 mem_gb:52.82
|
| 160 |
+
Training: 93%|█████████▎| 28000/30000 [28:18:43<1:58:20, 3.55s/step]INFO 2026-07-22 18:08:09 ot_train.py:646 step:58K smpl:7M ep:168K epch:34.00 loss:0.006 grdn:0.080 lr:3.1e-07 updt_s:2.388 data_s:1.179 smp/s:36 mem_gb:52.82
|
| 161 |
+
Training: 94%|█████████▍| 28200/30000 [28:30:35<1:46:58, 3.57s/step]INFO 2026-07-22 18:20:02 ot_train.py:646 step:58K smpl:7M ep:168K epch:34.12 loss:0.006 grdn:0.085 lr:2.5e-07 updt_s:2.385 data_s:1.176 smp/s:36 mem_gb:52.82
|
| 162 |
+
Training: 95%|█████████▍| 28400/30000 [28:42:28<1:34:56, 3.56s/step]INFO 2026-07-22 18:31:55 ot_train.py:646 step:58K smpl:7M ep:169K epch:34.23 loss:0.006 grdn:0.081 lr:2.0e-07 updt_s:2.386 data_s:1.177 smp/s:36 mem_gb:52.82
|
| 163 |
+
Training: 95%|█████████▌| 28600/30000 [28:54:23<1:24:22, 3.62s/step]INFO 2026-07-22 18:43:49 ot_train.py:646 step:59K smpl:8M ep:169K epch:34.35 loss:0.006 grdn:0.078 lr:1.6e-07 updt_s:2.392 data_s:1.178 smp/s:36 mem_gb:52.82
|
| 164 |
+
Training: 96%|█████████▌| 28800/30000 [29:06:18<1:11:24, 3.57s/step]INFO 2026-07-22 18:55:44 ot_train.py:646 step:59K smpl:8M ep:170K epch:34.47 loss:0.006 grdn:0.080 lr:1.2e-07 updt_s:2.381 data_s:1.191 smp/s:36 mem_gb:52.74
|
| 165 |
+
Training: 97%|█████████▋| 29000/30000 [29:18:11<59:41, 3.58s/step]INFO 2026-07-22 19:07:37 ot_train.py:646 step:59K smpl:8M ep:170K epch:34.58 loss:0.006 grdn:0.077 lr:8.4e-08 updt_s:2.388 data_s:1.175 smp/s:36 mem_gb:52.82
|
| 166 |
+
Training: 97%|█████████▋| 29200/30000 [29:30:03<47:36, 3.57s/step]INFO 2026-07-22 19:19:29 ot_train.py:646 step:59K smpl:8M ep:171K epch:34.70 loss:0.006 grdn:0.077 lr:5.7e-08 updt_s:2.385 data_s:1.174 smp/s:36 mem_gb:52.82
|
| 167 |
+
Training: 98%|█████████▊| 29400/30000 [29:41:56<35:29, 3.55s/step]INFO 2026-07-22 19:31:22 ot_train.py:646 step:59K smpl:8M ep:172K epch:34.82 loss:0.006 grdn:0.082 lr:3.4e-08 updt_s:2.386 data_s:1.178 smp/s:36 mem_gb:52.82
|
| 168 |
+
Training: 99%|█████████▊| 29600/30000 [29:53:47<23:38, 3.55s/step]INFO 2026-07-22 19:43:14 ot_train.py:646 step:60K smpl:8M ep:172K epch:34.94 loss:0.006 grdn:0.075 lr:1.8e-08 updt_s:2.381 data_s:1.173 smp/s:36 mem_gb:52.82
|
| 169 |
+
Training: 99%|█████████▉| 29800/30000 [30:05:40<11:44, 3.52s/step]INFO 2026-07-22 19:55:06 ot_train.py:646 step:60K smpl:8M ep:173K epch:35.05 loss:0.006 grdn:0.081 lr:6.5e-09 updt_s:2.385 data_s:1.174 smp/s:36 mem_gb:52.82
|
| 170 |
+
Training: 100%|██████████| 30000/30000 [30:17:33<00:00, 3.56s/step]INFO 2026-07-22 20:06:59 ot_train.py:646 step:60K smpl:8M ep:173K epch:35.17 loss:0.006 grdn:0.078 lr:9.2e-10 updt_s:2.378 data_s:1.187 smp/s:36 mem_gb:52.74
|
| 171 |
+
INFO 2026-07-22 20:06:59 ot_train.py:691 Checkpoint policy after step 60000
|
| 172 |
+
INFO 2026-07-22 20:07:27 in_yaml.py:1245 Saved config, action-space, prompt, and phase provenance in /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline and /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/060000
|
| 173 |
+
Training: 100%|██████████| 30000/30000 [30:18:01<00:00, 3.64s/step]
|
| 174 |
+
INFO 2026-07-22 20:07:27 ot_train.py:777 End of training
|
Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/requirements.txt
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
multiprocess==0.70.19
|
| 2 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 3 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 4 |
+
nvidia-nvjitlink==13.0.88
|
| 5 |
+
numpy==2.2.6
|
| 6 |
+
shellingham==1.5.4
|
| 7 |
+
pip==26.1.2
|
| 8 |
+
zipp==4.1.0
|
| 9 |
+
mypy_extensions==1.1.0
|
| 10 |
+
pillow==12.3.0
|
| 11 |
+
wheel==0.47.0
|
| 12 |
+
cuda-pathfinder==1.5.6
|
| 13 |
+
dm-tree==0.1.10
|
| 14 |
+
hf-xet==1.5.1
|
| 15 |
+
packaging==25.0
|
| 16 |
+
typing-inspect==0.9.0
|
| 17 |
+
nvidia-cuda-runtime==13.0.96
|
| 18 |
+
pyyaml-include==1.4.1
|
| 19 |
+
wrapt==2.2.2
|
| 20 |
+
pytz==2026.2
|
| 21 |
+
Pygments==2.20.0
|
| 22 |
+
urllib3==2.7.0
|
| 23 |
+
yarl==1.24.2
|
| 24 |
+
regex==2026.7.10
|
| 25 |
+
anyio==4.14.2
|
| 26 |
+
timm==1.0.28
|
| 27 |
+
typer==0.26.8
|
| 28 |
+
tokenizers==0.22.2
|
| 29 |
+
lerobot==0.6.0
|
| 30 |
+
annotated-doc==0.0.4
|
| 31 |
+
importlib_metadata==9.0.0
|
| 32 |
+
idna==3.18
|
| 33 |
+
mpmath==1.3.0
|
| 34 |
+
platformdirs==4.10.0
|
| 35 |
+
decord==0.6.0
|
| 36 |
+
transformers==5.5.4
|
| 37 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 38 |
+
accelerate==1.14.0
|
| 39 |
+
torchvision==0.26.0
|
| 40 |
+
torchcodec==0.11.1
|
| 41 |
+
nvidia-cufft==12.0.0.61
|
| 42 |
+
mdurl==0.1.2
|
| 43 |
+
sentry-sdk==2.64.0
|
| 44 |
+
av==15.1.0
|
| 45 |
+
nvidia-cusparse==12.6.3.3
|
| 46 |
+
PyYAML==6.0.3
|
| 47 |
+
python-dateutil==2.9.0.post0
|
| 48 |
+
protobuf==7.35.1
|
| 49 |
+
cmake==4.1.3
|
| 50 |
+
pydantic_core==2.46.4
|
| 51 |
+
requests==2.34.2
|
| 52 |
+
typing-inspection==0.4.2
|
| 53 |
+
nvidia-cufile==1.15.1.6
|
| 54 |
+
MarkupSafe==3.0.3
|
| 55 |
+
wandb==0.27.2
|
| 56 |
+
smmap==5.0.3
|
| 57 |
+
nvidia-cuda-cupti==13.0.85
|
| 58 |
+
charset-normalizer==3.4.9
|
| 59 |
+
h11==0.16.0
|
| 60 |
+
mergedeep==1.3.4
|
| 61 |
+
opencv-python-headless==4.13.0.92
|
| 62 |
+
draccus==0.10.0
|
| 63 |
+
annotated-types==0.7.0
|
| 64 |
+
rich==15.0.0
|
| 65 |
+
networkx==3.6.1
|
| 66 |
+
GitPython==3.1.51
|
| 67 |
+
safetensors==0.8.0
|
| 68 |
+
markdown-it-py==4.2.0
|
| 69 |
+
gitdb==4.0.12
|
| 70 |
+
toml==0.10.2
|
| 71 |
+
filelock==3.29.7
|
| 72 |
+
diffusers==0.35.2
|
| 73 |
+
typing_extensions==4.16.0
|
| 74 |
+
certifi==2026.6.17
|
| 75 |
+
tzdata==2026.3
|
| 76 |
+
nvidia-nccl-cu13==2.28.9
|
| 77 |
+
aiohappyeyeballs==2.7.1
|
| 78 |
+
setuptools==80.10.2
|
| 79 |
+
sympy==1.14.0
|
| 80 |
+
torch==2.11.0
|
| 81 |
+
einops==0.8.2
|
| 82 |
+
nvidia-curand==10.4.0.35
|
| 83 |
+
httpx==0.28.1
|
| 84 |
+
nvidia-cudnn-cu13==9.19.0.56
|
| 85 |
+
frozenlist==1.8.0
|
| 86 |
+
termcolor==3.3.0
|
| 87 |
+
multidict==6.7.1
|
| 88 |
+
httpcore==1.0.9
|
| 89 |
+
psutil==7.2.2
|
| 90 |
+
fsspec==2026.2.0
|
| 91 |
+
dill==0.4.1
|
| 92 |
+
pydantic==2.13.4
|
| 93 |
+
cuda-bindings==13.3.1
|
| 94 |
+
cloudpickle==3.1.2
|
| 95 |
+
aiohttp==3.14.1
|
| 96 |
+
datasets==4.8.5
|
| 97 |
+
gymnasium==1.3.0
|
| 98 |
+
jsonlines==4.0.0
|
| 99 |
+
pyarrow==25.0.0
|
| 100 |
+
attrs==26.1.0
|
| 101 |
+
peft==0.19.1
|
| 102 |
+
nvidia-cublas==13.1.0.3
|
| 103 |
+
propcache==0.5.2
|
| 104 |
+
Jinja2==3.1.6
|
| 105 |
+
aiosignal==1.4.0
|
| 106 |
+
nvidia-nvtx==13.0.85
|
| 107 |
+
triton==3.6.0
|
| 108 |
+
tqdm==4.68.4
|
| 109 |
+
pandas==2.3.3
|
| 110 |
+
click==8.4.2
|
| 111 |
+
Farama-Notifications==0.0.6
|
| 112 |
+
six==1.17.0
|
| 113 |
+
huggingface_hub==1.23.0
|
| 114 |
+
nvidia-cusolver==12.0.4.66
|
| 115 |
+
cuda-toolkit==13.0.2
|
| 116 |
+
xxhash==3.8.1
|
| 117 |
+
absl-py==2.5.0
|
Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/files/wandb-summary.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_timestamp":1.7847184199444346e+09,"train/lr":9.22312585679652e-10,"train/samples":7680000,"train/samples_per_s":35.90449332836475,"train/episodes":173388.83622525382,"_step":60000,"train/update_s":2.378391921520233,"_wandb":{"runtime":229206},"train/epochs":35.17014933575128,"train/loss":0.006419091485440731,"train/dataloading_s":1.1866214830521493,"train/gpu_mem_gb":52.739501359462736,"_runtime":229206,"train/steps":60000,"train/grad_norm":0.07766221214085817}
|
Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug-core.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-07-21T13:48:50.453532199+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9uv2jmwf/port-1492997.txt","pid":1492997,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-07-21T13:48:50.454087052+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1492997}
|
| 3 |
+
{"time":"2026-07-21T13:48:50.454079632+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1492997-1493484-4220890972/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-07-21T13:48:50.633551268+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-07-21T13:48:50.660462299+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"h160hqm2","id":"1(@)"}
|
| 6 |
+
{"time":"2026-07-21T13:48:51.096143655+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"h160hqm2","id":"1(@)"}
|
| 7 |
+
{"time":"2026-07-21T13:48:57.080549699+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6hzdf802emtu"}
|
| 8 |
+
{"time":"2026-07-22T20:07:27.908582433+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-07-22T20:07:27.908705751+09:00","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-07-22T20:07:27.908732237+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 11 |
+
{"time":"2026-07-22T20:07:27.908707498+09:00","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
|
| 12 |
+
{"time":"2026-07-22T20:07:27.90880098+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 13 |
+
{"time":"2026-07-22T20:07:27.908796618+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1492997-1493484-4220890972/socket","Net":"unix"}}
|
| 14 |
+
{"time":"2026-07-22T20:08:09.504153168+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 15 |
+
{"time":"2026-07-22T20:08:09.504205831+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 16 |
+
{"time":"2026-07-22T20:08:09.504234606+09:00","level":"INFO","msg":"server is closed"}
|
Abs_6D + DeltaEE/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug.log
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
|
| 2 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_setup.py:_flush():81] Configure stats pid to 1492997
|
| 3 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug.log
|
| 5 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260721_134850-h160hqm2/logs/debug-internal.log
|
| 6 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:init():766] calling init triggers
|
| 7 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/seonho/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [16]}, 'observation.images.wrist': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [16]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'STATE': <NormalizationMode.IDENTITY: 'IDENTITY'>, 'ACTION': <NormalizationMode.IDENTITY: 'IDENTITY'>}, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'h160hqm2', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/seonho/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/checkpoints/030000', '_wandb': {}}
|
| 9 |
+
2026-07-21 13:48:50,416 INFO MainThread:1492997 [wandb_init.py:init():814] starting backend
|
| 10 |
+
2026-07-21 13:48:50,633 INFO MainThread:1492997 [wandb_init.py:init():829] sending inform_init request
|
| 11 |
+
2026-07-21 13:48:51,097 INFO MainThread:1492997 [wandb_init.py:init():834] backend started and connected
|
| 12 |
+
2026-07-21 13:48:51,099 INFO MainThread:1492997 [wandb_init.py:init():904] updated telemetry
|
| 13 |
+
2026-07-21 13:48:51,105 INFO MainThread:1492997 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-07-21 13:48:51,906 INFO MainThread:1492997 [wandb_init.py:init():967] run resumed
|
| 15 |
+
2026-07-21 13:48:51,912 INFO MainThread:1492997 [wandb_init.py:init():972] starting run threads in backend
|
| 16 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_console_start():2523] atexit reg
|
| 17 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 18 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 19 |
+
2026-07-21 13:48:52,047 INFO MainThread:1492997 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 20 |
+
2026-07-21 13:48:52,050 INFO MainThread:1492997 [wandb_init.py:init():1010] run started, returning control to user process
|
| 21 |
+
2026-07-22 20:07:27,908 INFO wandb-AsyncioManager-main:1492997 [service_client.py:_forward_responses():122] Reached EOF.
|
| 22 |
+
2026-07-22 20:07:27,909 INFO wandb-AsyncioManager-main:1492997 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles.
|
| 23 |
+
2026-07-22 20:07:27,910 ERROR wandb-AsyncioManager-main:1492997 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
|
| 24 |
+
Traceback (most recent call last):
|
| 25 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
|
| 26 |
+
await fn()
|
| 27 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop
|
| 28 |
+
await asyncio_compat.race(
|
| 29 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race
|
| 30 |
+
async with open_task_group(race=True) as tg:
|
| 31 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 32 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__
|
| 33 |
+
await anext(self.gen)
|
| 34 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group
|
| 35 |
+
await task_group._wait_all(race=race, timeout=exit_timeout)
|
| 36 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all
|
| 37 |
+
raise exc
|
| 38 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all
|
| 39 |
+
result = await handle.wait_async(timeout=None)
|
| 40 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 41 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async
|
| 42 |
+
response = await self._handle.wait_async(timeout=timeout)
|
| 43 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 44 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async
|
| 45 |
+
raise HandleAbandonedError()
|
| 46 |
+
wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError
|
| 47 |
+
2026-07-22 20:07:33,323 ERROR wandb-AsyncioManager-main:1492997 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback.
|
| 48 |
+
Traceback (most recent call last):
|
| 49 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
|
| 50 |
+
await fn()
|
| 51 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop
|
| 52 |
+
should_stop = await self._check_should_stop()
|
| 53 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 54 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop
|
| 55 |
+
handle = await self._interface.deliver_async(request)
|
| 56 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 57 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async
|
| 58 |
+
handle = await self._client.deliver(request)
|
| 59 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 60 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver
|
| 61 |
+
handle = self._mailbox.require_response(request)
|
| 62 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 63 |
+
File "/home/seonho/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response
|
| 64 |
+
raise MailboxClosedError()
|
| 65 |
+
wandb.sdk.mailbox.mailbox.MailboxClosedError
|